[WIP] Generate duplicate maps

grendello · grendello · commit 690efa6a3ae1 · 2020-01-13T20:46:04.000+01:00
diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeTypeMappingData.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeTypeMappingData.cs
@@ -33,9 +33,10 @@ public NativeTypeMappingData (IDictionary<byte[], TypeMapGenerator.ModuleData> m
 				AssemblyNames.Add (data.AssemblyNameLabel, data.AssemblyName);
 
 				int moduleIndex = moduleList.IndexOf (data);
-				foreach (TypeMapGenerator.TypeMapEntry entry in data.Types) {
+				foreach (var kvp2 in data.Types) {
+					TypeMapGenerator.TypeMapEntry entry = kvp2.Value;
 					entry.ModuleIndex = moduleIndex;
-					JavaTypes[entry.JavaName] = entry;
+					JavaTypes.Add (entry.JavaName, entry);
 				}
 			}
 		}
diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs
@@ -48,7 +48,8 @@ internal sealed class ModuleData
 		{
 			public Guid Mvid;
 			public AssemblyDefinition Assembly;
-			public List<TypeMapEntry> Types;
+			public SortedDictionary<string, TypeMapEntry> Types;
+			public Dictionary<uint, TypeMapEntry> DuplicateTypes;
 			public string AssemblyName;
 			public string AssemblyNameLabel;
 			public string OutputFilePath;
@@ -124,7 +125,8 @@ public bool Generate (DirectoryAssemblyResolver resolver, IEnumerable<string> as
 						Mvid = td.Module.Mvid,
 						Assembly = td.Module.Assembly,
 						AssemblyName = td.Module.Assembly.Name.Name,
-						Types = new List<TypeMapEntry> (),
+						Types = new SortedDictionary<string, TypeMapEntry> (StringComparer.Ordinal),
+						DuplicateTypes = new Dictionary<uint, TypeMapEntry> (),
 					};
 					modules.Add (moduleUUID, moduleData);
 
@@ -151,14 +153,18 @@ public bool Generate (DirectoryAssemblyResolver resolver, IEnumerable<string> as
 						maxJavaNameLength = javaName.Length;
 				}
 
-				moduleData.Types.Add (
-					new TypeMapEntry {
-						JavaName = javaName,
-						ManagedTypeName = td.FullName,
-						Token = td.MetadataToken.ToUInt32 (),
-						AssemblyNameIndex = knownAssemblies [assemblyName]
-					}
-				);
+				var entry = new TypeMapEntry {
+					JavaName = javaName,
+					ManagedTypeName = td.FullName,
+					Token = td.MetadataToken.ToUInt32 (),
+					AssemblyNameIndex = knownAssemblies [assemblyName]
+				};
+
+				if (moduleData.Types.ContainsKey (entry.JavaName)) {
+					logger ($"Warning: duplicate Java type name '{entry.JavaName}' (new token: {entry.Token}).");
+					moduleData.DuplicateTypes.Add (entry.Token, entry);
+				} else
+					moduleData.Types.Add (entry.JavaName, entry);
 			}
 
 			NativeTypeMappingData data;
@@ -269,15 +275,17 @@ void OutputModule (string outputDirectory, byte[] moduleUUID, ModuleData moduleD
 
 		// Binary file format, all data is little-endian:
 		//
-		//  [Magic string]           # XATM
-		//  [Format version]         # 32-bit integer, 4 bytes
-		//  [Module UUID]            # 16 bytes
-		//  [Entry count]            # 32-bit integer, 4 bytes
-		//  [Java type name width]   # 32-bit integer, 4 bytes
-		//  [Assembly name size]     # 32-bit integer, 4 bytes
-		//  [Assembly name]          # Non-null terminated assembly name
-		//  [Java-to-managed map]    # Format described below, [Entry count] entries
-		//  [Managed-to-java map]    # Format described below, [Entry count] entries
+		//  [Magic string]                    # XATM
+		//  [Format version]                  # 32-bit integer, 4 bytes
+		//  [Module UUID]                     # 16 bytes
+		//  [Entry count]                     # 32-bit integer, 4 bytes
+		//  [Duplicate count]                 # 32-bit integer, 4 bytes (might be 0)
+		//  [Java type name width]            # 32-bit integer, 4 bytes
+		//  [Assembly name size]              # 32-bit integer, 4 bytes
+		//  [Assembly name]                   # Non-null terminated assembly name
+		//  [Java-to-managed map]             # Format described below, [Entry count] entries
+		//  [Managed-to-java map]             # Format described below, [Entry count] entries
+		//  [Managed-to-java duplicates map]  # Map of unique managed IDs which point to the same Java type name (might be empty)
 		//
 		// Java-to-managed map format:
 		//
@@ -297,27 +305,22 @@ void OutputModule (string outputDirectory, byte[] moduleUUID, ModuleData moduleD
 		//  Both fields are 32-bit integers, to a total of 8 bytes per entry. Index points into the
 		//  [Java-to-managed map] table above.
 		//
+		// Managed-to-java duplicates map format:
+		//
+		//  Format is identical to [Managed-to-java] above.
+		//
 		void OutputModule (BinaryWriter bw, byte[] moduleUUID, ModuleData moduleData)
 		{
 			bw.Write (moduleMagicString);
 			bw.Write (TypeMapFormatVersion);
 			bw.Write (moduleUUID);
 
 			var javaNames = new SortedDictionary<string, uint> (StringComparer.Ordinal);
-			var duplicateJavaNames = new Dictionary<string, bool> (StringComparer.Ordinal);
 			var managedTypes = new SortedDictionary<uint, uint> ();
-
-			// TODO: put duplicate types in a separate list (typeid:index) at the end of the typemap file/assembly code
-			//       and mark such entries somehow in the typeid->java table (e.g. set the high bit of the index, it's unlikely
-			//       we'd have 4 billion entries)
 			int maxJavaNameLength = 0;
-			foreach (TypeMapEntry entry in moduleData.Types) {
-				if (javaNames.ContainsKey (entry.JavaName)) {
-					logger ($"Warning: duplicate Java type name '{entry.JavaName}' (old token: {javaNames [entry.JavaName]}; new token: {entry.Token}). Ignoring the new type.");
-					if (!duplicateJavaNames.ContainsKey (entry.JavaName))
-						duplicateJavaNames.Add (entry.JavaName, false);
-					continue;
-				}
+
+			foreach (var kvp in moduleData.Types) {
+				TypeMapEntry entry = kvp.Value;
 
 				javaNames.Add (entry.JavaName, entry.Token);
 				if (entry.JavaName.Length > maxJavaNameLength)
@@ -327,17 +330,14 @@ void OutputModule (BinaryWriter bw, byte[] moduleUUID, ModuleData moduleData)
 			}
 
 			var javaNameList = javaNames.Keys.ToList ();
-			foreach (TypeMapEntry entry in moduleData.Types) {
-				if (duplicateJavaNames.TryGetValue (entry.JavaName, out bool skip)) {
-					if (skip)
-						continue;
-					duplicateJavaNames [entry.JavaName] = true;
-				}
-
-				managedTypes [entry.Token] = (uint)javaNameList.IndexOf (entry.JavaName);
+			foreach (var kvp in moduleData.Types) {
+				TypeMapEntry entry = kvp.Value;
+				var javaIndex = (uint)javaNameList.IndexOf (entry.JavaName);
+				managedTypes.Add (entry.Token, javaIndex);
 			}
 
 			bw.Write (javaNames.Count);
+			bw.Write (moduleData.DuplicateTypes.Count);
 			bw.Write (maxJavaNameLength + 1);
 
 			string assemblyName = moduleData.Assembly.Name.Name;
@@ -353,12 +353,29 @@ void OutputModule (BinaryWriter bw, byte[] moduleUUID, ModuleData moduleData)
 				bw.Write (token);
 			}
 
-			foreach (var kvp in managedTypes) {
-				uint token = kvp.Key;
-				uint javaIndex = kvp.Value;
+			WriteManagedTypes (managedTypes);
+			if (moduleData.DuplicateTypes.Count == 0)
+				return;
 
-				bw.Write (token);
-				bw.Write (javaIndex);
+			var managedDuplicates = new SortedDictionary<uint, uint> ();
+			foreach (var kvp in moduleData.DuplicateTypes) {
+				uint javaIndex = kvp.Key;
+				uint typeId = kvp.Value.Token;
+
+				managedDuplicates.Add (javaIndex, typeId);
+			}
+
+			WriteManagedTypes (managedDuplicates);
+
+			void WriteManagedTypes (IDictionary<uint, uint> types)
+			{
+				foreach (var kvp in types) {
+					uint token = kvp.Key;
+					uint javaIndex = kvp.Value;
+
+					bw.Write (token);
+					bw.Write (javaIndex);
+				}
 			}
 		}
 
diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingNativeAssemblyGenerator.cs
@@ -99,11 +99,15 @@ void WriteAssemblyNames (StreamWriter output)
 			}
 		}
 
-		void WriteManagedMaps (StreamWriter output, string moduleSymbolName, TypeMapGenerator.ModuleData data)
+		void WriteManagedMaps<TKey> (StreamWriter output, string moduleSymbolName, IDictionary<TKey, TypeMapGenerator.TypeMapEntry> entries)
 		{
+			if (entries == null || entries.Count == 0)
+				return;
+
 			var javaTypes = mappingData.JavaTypes.Keys.ToList ();
 			var tokens = new SortedDictionary<uint, uint> ();
-			foreach (TypeMapGenerator.TypeMapEntry entry in data.Types) {
+			foreach (var kvp in entries) {
+				TypeMapGenerator.TypeMapEntry entry = kvp.Value;
 				tokens[entry.Token] = (uint)javaTypes.IndexOf (entry.JavaName);
 			}
 
@@ -143,17 +147,26 @@ void WriteMapModules (StreamWriter output, StreamWriter mapOutput, string symbol
 				TypeMapGenerator.ModuleData data = kvp.Value;
 
 				string mapName = $"module{moduleCounter++}_managed_to_java";
-				size += WriteStructure (output, fieldAlignBytes, () => WriteMapModule (output, mapName, mvid, data));
-				if (mapOutput != null)
-					WriteManagedMaps (mapOutput, mapName, data);
+				string duplicateMapName;
+
+				if (data.DuplicateTypes.Count == 0)
+					duplicateMapName = null;
+				else
+					duplicateMapName = $"{mapName}_duplicates";
+
+				size += WriteStructure (output, fieldAlignBytes, () => WriteMapModule (output, mapName, duplicateMapName, mvid, data));
+				if (mapOutput != null) {
+					WriteManagedMaps (mapOutput, mapName, data.Types);
+					WriteManagedMaps (mapOutput, duplicateMapName, data.DuplicateTypes);
+				}
 			}
 
 			WriteStructureSize (output, symbolName, size);
 			WriteCommentLine (output, "Managed to Java map: END", indent: false);
 			output.WriteLine ();
 		}
 
-		uint WriteMapModule (StreamWriter output, string mapName, byte[] mvid, TypeMapGenerator.ModuleData data)
+		uint WriteMapModule (StreamWriter output, string mapName, string duplicateMapName, byte[] mvid, TypeMapGenerator.ModuleData data)
 		{
 			uint size = 0;
 			WriteCommentLine (output, $"module_uuid: {data.Mvid}");
@@ -162,9 +175,15 @@ uint WriteMapModule (StreamWriter output, string mapName, byte[] mvid, TypeMapGe
 			WriteCommentLine (output, "entry_count");
 			size += WriteData (output, data.Types.Count);
 
+			WriteCommentLine (output, "duplicate_count");
+			size += WriteData (output, data.DuplicateTypes.Count);
+
 			WriteCommentLine (output, "map");
 			size += WritePointer (output, mapName);
 
+			WriteCommentLine (output, "duplicate_map");
+			size += WritePointer (output, duplicateMapName);
+
 			WriteCommentLine (output, $"assembly_name: {data.AssemblyName}");
 			size += WritePointer (output, MakeLocalLabel (data.AssemblyNameLabel));
 
diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc
@@ -602,27 +602,12 @@ EmbeddedAssemblies::typemap_load_index (int dir_fd, const char *dir_path, const
 }
 
 bool
-EmbeddedAssemblies::typemap_load_file (int dir_fd, const char *dir_path, const char *file_path, TypeMapModule &module)
+EmbeddedAssemblies::typemap_load_file (BinaryTypeMapHeader &header, const char *dir_path, const char *file_path, int file_fd, TypeMapModule &module)
 {
-	log_debug (LOG_ASSEMBLY, "Loading TypeMap file '%s/%s'", dir_path, file_path);
-
-	bool ret = true;
-	BinaryTypeMapHeader header;
-	size_t file_size;
-	int fd = -1;
-	size_t alloc_size;
-	ssize_t nread;;
-	char *chars;
-	uint8_t *p;
-
-	if (!typemap_read_header (dir_fd, "TypeMap", dir_path, file_path, MODULE_MAGIC, header, file_size, fd)) {
-		ret = false;
-		goto cleanup;
-	}
-
-	alloc_size = ADD_WITH_OVERFLOW_CHECK (size_t, header.assembly_name_length, 1);
+	size_t alloc_size = ADD_WITH_OVERFLOW_CHECK (size_t, header.assembly_name_length, 1);
 	module.assembly_name = new char[alloc_size];
-	nread = read (fd, module.assembly_name, header.assembly_name_length);
+
+	ssize_t nread = read (file_fd, module.assembly_name, header.assembly_name_length);
 	module.assembly_name [header.assembly_name_length] = 0;
 	module.entry_count = header.entry_count;
 
@@ -632,31 +617,23 @@ EmbeddedAssemblies::typemap_load_file (int dir_fd, const char *dir_path, const c
 	alloc_size = MULTIPLY_WITH_OVERFLOW_CHECK (size_t, header.java_name_width + 4, header.entry_count);
 	module.java_name_width = header.java_name_width;
 	module.java_map = new uint8_t[alloc_size];
-	nread = read (fd, module.java_map, alloc_size);
+	nread = read (file_fd, module.java_map, alloc_size);
 	if (nread != static_cast<ssize_t>(alloc_size)) {
 		log_error (LOG_ASSEMBLY, "Failed to read %u bytes (java-to-managed) from module file %s/%s. %s", alloc_size, dir_path, file_path, strerror (errno));
-		ret = false;
-		delete[] module.java_map;
-		module.java_map = nullptr;
-		goto cleanup;
+		return false;
 	}
 
 	module.map = new TypeMapModuleEntry[header.entry_count];
 	alloc_size = MULTIPLY_WITH_OVERFLOW_CHECK (size_t, sizeof(TypeMapModuleEntry), header.entry_count);
-	nread = read (fd, module.map, alloc_size);
+	nread = read (file_fd, module.map, alloc_size);
 	if (nread != static_cast<ssize_t>(alloc_size)) {
 		log_error (LOG_ASSEMBLY, "Failed to read %u bytes (managed-to-java) from module file %s/%s. %s", alloc_size, dir_path, file_path, strerror (errno));
-		ret = false;
-		delete[] module.java_map;
-		module.java_map = nullptr;
-		delete[] module.map;
-		module.map = nullptr;
-		goto cleanup;
+		return false;
 	}
 
 	alloc_size = module.java_name_width + 1;
-	chars = new char[alloc_size]();
-	p = module.java_map;
+	auto chars = new char[alloc_size]();
+	uint8_t *p = module.java_map;
 	log_debug (LOG_ASSEMBLY, "Java entries in %s/%s", dir_path, file_path);
 	for (size_t i = 0; i < module.entry_count; i++) {
 		memcpy (chars, p, module.java_name_width);
@@ -671,10 +648,40 @@ EmbeddedAssemblies::typemap_load_file (int dir_fd, const char *dir_path, const c
 		log_debug (LOG_ASSEMBLY, "  %04u: token %u (0x%x); index %u", i, module.map[i].type_token_id, module.map[i].type_token_id, module.map[i].java_map_index);
 	}
 
+	return true;
+}
+
+bool
+EmbeddedAssemblies::typemap_load_file (int dir_fd, const char *dir_path, const char *file_path, TypeMapModule &module)
+{
+	log_debug (LOG_ASSEMBLY, "Loading TypeMap file '%s/%s'", dir_path, file_path);
+
+	bool ret = true;
+	BinaryTypeMapHeader header;
+	size_t file_size;
+	int fd = -1;
+
+	module.java_map = nullptr;
+	module.map = nullptr;
+
+	if (!typemap_read_header (dir_fd, "TypeMap", dir_path, file_path, MODULE_MAGIC, header, file_size, fd)) {
+		ret = false;
+		goto cleanup;
+	}
+
+	ret = typemap_load_file (header, dir_path, file_path, fd, module);
+
   cleanup:
 	if (fd >= 0)
 		close (fd);
 
+	if (!ret) {
+		delete[] module.java_map;
+		module.java_map = nullptr;
+		delete[] module.map;
+		module.map = nullptr;
+	}
+
 	return ret;
 }
 
diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh
@@ -78,6 +78,7 @@ namespace xamarin::android::internal {
 		uint8_t* typemap_load_index (int dir_fd, const char *dir_path, const char *index_path);
 		uint8_t* typemap_load_index (TypeMapIndexHeader &header, size_t file_size, int index_fd);
 		bool typemap_load_file (int dir_fd, const char *dir_path, const char *file_path, TypeMapModule &module);
+		bool typemap_load_file (BinaryTypeMapHeader &header, const char *dir_path, const char *file_path, int file_fd, TypeMapModule &module);
 #endif // DEBUG || !ANDROID
 		bool register_debug_symbols_for_assembly (const char *entry_name, MonoBundledAssembly *assembly, const mono_byte *debug_contents, int debug_size);
 
diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh
@@ -18,6 +18,7 @@ struct BinaryTypeMapHeader
 	uint32_t version;
 	uint8_t  module_uuid[16];
 	uint32_t entry_count;
+	uint32_t duplicate_count;
 	uint32_t java_name_width;
 	uint32_t assembly_name_length;
 };
@@ -41,6 +42,7 @@ struct TypeMapModule
 	uint8_t                   module_uuid[16];
 	uint32_t                  entry_count;
 	TypeMapModuleEntry       *map;
+	TypeMapModuleEntry       *duplicate_map;
 	char                     *assembly_name;
 	MonoImage                *image;
 #ifdef DEBUG

Original file line number	Diff line number	Diff line change
`@@ -33,9 +33,10 @@ public NativeTypeMappingData (IDictionary<byte[], TypeMapGenerator.ModuleData> m`
`33`	`33`	`AssemblyNames.Add (data.AssemblyNameLabel, data.AssemblyName);`
`34`	`34`
`35`	`35`	`int moduleIndex = moduleList.IndexOf (data);`
`36`		`- foreach (TypeMapGenerator.TypeMapEntry entry in data.Types) {`
	`36`	`+ foreach (var kvp2 in data.Types) {`
	`37`	`+ TypeMapGenerator.TypeMapEntry entry = kvp2.Value;`
`37`	`38`	`entry.ModuleIndex = moduleIndex;`
`38`		`- JavaTypes[entry.JavaName] = entry;`
	`39`	`+ JavaTypes.Add (entry.JavaName, entry);`
`39`	`40`	`}`
`40`	`41`	`}`
`41`	`42`	`}`