[class-parse] support Module AttributeInfo (#1097)

Fixes: #1096
Context: https://stackoverflow.com/questions/57358750/module-info-class-file-is-different-in-the-module-jar-file-and-compiled-module-i
Context: 678c4bd
Context: b274a67
JDK 9 adds support for [modules][0], which are (kinda sorta) like
.NET Assemblies: modules can depend upon other modules, export
packages, etc.
In particular:
> **exports and exports…to.** An exports module directive specifies
> one of the module’s packages whose `public` types (and their nested
> `public` and `protected` types) should be accessible to code in all
> other modules.
This allows an equivalent to the [C# `internal` access modifier][1]:
`public` types in a *non-`export`ed package* should be treated as
"internal", while `public` types in an `export`ed package are
"fully public".
Update `Xamarin.Android.Tools.Bytecode.dll` to extract the module-
related information, then update `ClassPath` so that it updates all
`public` types *outside* of the "exported" packages to have an
`//*/@annotated-visibility` attribute value of `module-info`.
(See also commit b274a67, which added `//*/@@annotated-visibility`.)
If there is *already* an `//*/@annotated-visibility` value, then we
*append* ` module-info` to the attribute value.
We use `//*/@annotated-visibility` because we are concerned about
introducing an ABI break into AndroidX-related bindings because of
type visibility changes. If this isn't a concern, it should be
possible to use Metadata to remove those types:
<attr path="//class[@annotated-visibility]"
name="visibility">kotlin-internal</attr>
<attr path="//interface[@annotated-visibility]"
name="visibility">kotlin-internal</attr>
`class-parse` command-line parsing has been altered. There is now a
"global `ClassPath`", which will be used to hold `.class` files
provided on the command-line. `.jar` and `.jmod` files provided on
the command-line will be given their own `ClassPath` instances, and
`module-info.class`-based annotated-visibility fixups are specific to
each `ClassPath` instance. Global files are processed together.
There is thus no way for `module-info.class` visibility changes from
`a.jar` to impact `b.jar`. After visibilities are fixed up, we then
merge everything into the "global" `ClassPath` instance before
transforming to XML.
Additionally, `class-parse --dump` can now accept `.jar` files,
and will dump out *all* `.class` filers within the `.jar` file.
To make this output easier, each "entry" starts with a "header" of
`-- Begin {ClassFile.FullJniName}`, and a blank link will be printed
between each entry.
`tests/Xamarin.Android.Tools.Bytecode-Tests` has been updated to:
1. Contain a `module-info.java`, which declares a `com.xamarin`
module.
2. Add a new `com.xamarin.internal.PublicClassNotInModuleExports`
type which is *not* in the `com.xamarin` package, but instead
a *nested* package. The type is `public`.
3. Build a `xatb.jar` artifact
This makes for a simple one-off test:
% dotnet build tests/Xamarin.Android.Tools.Bytecode-Tests/*.csproj
% dotnet build tools/class-parse/*.csproj
% dotnet bin/Debug-net7.0/class-parse.dll \
tests/Xamarin.Android.Tools.Bytecode-Tests/obj/Debug-net7.0/xatb.jar
…
<class
name="PublicClassNotInModuleExports"
…
annotated-visibility="module-info" />
Note that `com.xamarin.internal.PublicClassNotInModuleExports` now
has an XML attribute `annotated-visibility="module-info"`.
Aside: the commit message of 678c4bd sadly overlooked this
[clarification][2] for why `kotlin-internal` was introduced:
> Note: we introduce and use a new `//*/@visibility` value of
> `kotlin-internal` because `internal` is an *existing* value that may
> be used in `Metadata.xml` files, e.g. making `public` API `internal`
> so that it can still be used in the binding, but isn't *public*.
Aside: a discovered oddity: `jar cf …` *modifies* `module-info.class`,
adding a `ModulePackages` attribute! (Specifically, if you compare
the "on-disk" `module-info.class` to the one within
`tests/Xamarin.Android.Tools.Bytecode-Tests/obj/$(Configuration)/xatb.jar`,
they differ in size!)
[0]: https://www.oracle.com/corporate/features/understanding-java-9-modules.html
[1]: https://learn.microsoft.com/en-us/dotnet/csharp/language-reference/keywords/internal
[2]: #793 (comment)

Author: jonpryor

src/Xamarin.Android.Tools.Bytecode/AttributeInfo.cs

@@ -44,6 +44,8 @@ public class AttributeInfo {
 publicconststringInnerClasses="InnerClasses";
 publicconststringLocalVariableTable="LocalVariableTable";
 publicconststringMethodParameters="MethodParameters";
+publicconststringModule="Module";
+publicconststringModulePackages="ModulePackages";
 publicconststringSignature="Signature";
 publicconststringSourceFile="SourceFile";
 publicconststringStackMapTable="StackMapTable";
@@ -79,6 +81,8 @@ public string Name {
 {typeof(InnerClassesAttribute),InnerClasses},
 {typeof(LocalVariableTableAttribute),LocalVariableTable},
 {typeof(MethodParametersAttribute),MethodParameters},
+{typeof(ModuleAttribute),Module},
+{typeof(ModulePackagesAttribute),ModulePackages},
 {typeof(RuntimeVisibleAnnotationsAttribute),RuntimeVisibleAnnotations},
 {typeof(RuntimeInvisibleAnnotationsAttribute),RuntimeInvisibleAnnotations},
 {typeof(SignatureAttribute),Signature},
@@ -98,6 +102,7 @@ internal static string GetAttributeName<T>()
 publicstaticAttributeInfoCreateFromStream(ConstantPoolconstantPool,Streamstream)
 {
 varnameIndex=stream.ReadNetworkUInt16();
+varconstant=constantPool[nameIndex];
 varname=((ConstantPoolUtf8Item)constantPool[nameIndex]).Value;
 varattr=CreateAttribute(name,constantPool,nameIndex,stream);
 returnattr;
@@ -114,6 +119,8 @@ static AttributeInfo CreateAttribute (string name, ConstantPool constantPool, us
 caseInnerClasses:returnnewInnerClassesAttribute(constantPool,nameIndex,stream);
 caseLocalVariableTable:returnnewLocalVariableTableAttribute(constantPool,nameIndex,stream);
 caseMethodParameters:returnnewMethodParametersAttribute(constantPool,nameIndex,stream);
+caseModule:returnnewModuleAttribute(constantPool,nameIndex,stream);
+caseModulePackages:returnnewModulePackagesAttribute(constantPool,nameIndex,stream);
 caseRuntimeVisibleAnnotations:returnnewRuntimeVisibleAnnotationsAttribute(constantPool,nameIndex,stream);
 caseRuntimeInvisibleAnnotations:returnnewRuntimeInvisibleAnnotationsAttribute(constantPool,nameIndex,stream);
 caseRuntimeInvisibleParameterAnnotations:returnnewRuntimeInvisibleParameterAnnotationsAttribute(constantPool,nameIndex,stream);
@@ -503,6 +510,118 @@ public override string ToString ()
 }
 }
 
+// https://docs.oracle.com/javase/specs/jvms/se11/html/jvms-4.html#jvms-4.7.25
+publicsealedclassModuleAttribute:AttributeInfo
+{
+ushortmoduleNameIndex;
+publicstringModuleName{
+get{return((ConstantPoolModuleItem)ConstantPool[moduleNameIndex]).Name.Value;}
+}
+
+publicModuleFlagsModuleFlags{get;privateset;}
+
+ushortmoduleVersionIndex;
+publicstring?ModuleVersion{
+get{return((ConstantPoolUtf8Item)ConstantPool[moduleVersionIndex])?.Value;}
+}
+
+publicCollection<ModuleRequiresInfo>Requires{get;}=new();
+publicCollection<ModuleExportsPackageInfo>Exports{get;}=new();
+publicCollection<ModuleOpensPackageInfo>Opens{get;}=new();
+publicCollection<ConstantPoolClassItem>Uses{get;}=new();
+publicCollection<ModuleProvidesInfo>Provides{get;}=new();
+
+publicModuleAttribute(ConstantPoolconstantPool,ushortnameIndex,Streamstream)
+:base(constantPool,nameIndex,stream)
+{
+varattribute_length=stream.ReadNetworkUInt32();
+
+moduleNameIndex=stream.ReadNetworkUInt16();
+ModuleFlags=(ModuleFlags)stream.ReadNetworkUInt16();
+moduleVersionIndex=stream.ReadNetworkUInt16();
+
+varrequires_count=stream.ReadNetworkUInt16();
+for(inti=0;i<requires_count;++i){
+Requires.Add(newModuleRequiresInfo(constantPool,stream));
+}
+
+varexports_count=stream.ReadNetworkUInt16();
+for(inti=0;i<exports_count;++i){
+Exports.Add(newModuleExportsPackageInfo(constantPool,stream));
+}
+
+varopens_count=stream.ReadNetworkUInt16();
+for(inti=0;i<opens_count;++i){
+Opens.Add(newModuleOpensPackageInfo(constantPool,stream));
+}
+
+varuses_count=stream.ReadNetworkUInt16();
+for(inti=0;i<uses_count;++i){
+varuses_index=stream.ReadNetworkUInt16();
+Uses.Add((ConstantPoolClassItem)constantPool[uses_index]);
+}
+
+varprovides_count=stream.ReadNetworkUInt16();
+for(inti=0;i<provides_count;++i){
+Provides.Add(newModuleProvidesInfo(constantPool,stream));
+}
+}
+
+publicoverridestringToString()
+{
+vars=newStringBuilder()
+.Append("Module(").AppendLine()
+.Append(" ").Append(nameof(ModuleName)).Append("='").Append(ModuleName).AppendLine("', ")
+.Append(" ").Append(nameof(ModuleVersion)).Append("='").Append(ModuleVersion).Append("'");
+AppendString(s,nameof(Requires),Requires);
+AppendString(s,nameof(Exports),Exports);
+AppendString(s,nameof(Opens),Opens);
+AppendString(s,nameof(Uses),Uses.Select(u =>$"UsesService({u.Name})").ToList());
+AppendString(s,nameof(Provides),Provides);
+s.Append(")");
+
+returns.ToString();
+}
+
+staticStringBuilderAppendString<T>(StringBuilders,stringcollectionName,IList<T>items)
+{
+if(items.Count==0){
+returns;
+}
+s.AppendLine(",");
+s.Append(" ").Append(collectionName).AppendLine("={");
+s.Append(" ").Append(items[0]);
+for(inti=1;i<items.Count;++i){
+s.AppendLine(",");
+s.Append(" ");
+s.Append(items[i]);
+}
+returns.Append("}");
+}
+}
+
+// https://docs.oracle.com/javase/specs/jvms/se11/html/jvms-4.html#jvms-4.7.26
+publicsealedclassModulePackagesAttribute:AttributeInfo{
+publicCollection<ConstantPoolPackageItem>Packages{get;}=new();
+
+publicModulePackagesAttribute(ConstantPoolconstantPool,ushortnameIndex,Streamstream)
+:base(constantPool,nameIndex,stream)
+{
+varattribute_length=stream.ReadNetworkUInt32();
+
+varpackage_count=stream.ReadNetworkUInt16();
+for(inti=0;i<package_count;++i){
+varpackage_index=stream.ReadNetworkUInt16();
+Packages.Add((ConstantPoolPackageItem)constantPool[package_index]);
+}
+}
+
+publicoverridestringToString()
+{
+return$"ModulePackages({{{string.Join (",", Packages.Select (p => p.Name.Value))}}})";
+}
+}
+
 // https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-4.html#jvms-4.7.16
 publicsealedclassRuntimeVisibleAnnotationsAttribute:AttributeInfo
 {

src/Xamarin.Android.Tools.Bytecode/ClassFile.cs

@@ -48,8 +48,15 @@ public ClassFile (Stream stream)
 Attributes=newAttributeCollection(ConstantPool,stream);
 
 inte=stream.ReadByte();
-if(e>=0)
-thrownewBadImageFormatException("Stream has trailing data?!");
+if(e>=0){
+vartrailing=newSystem.Text.StringBuilder();
+trailing.AppendFormat("{0:x2}",(byte)e);
+while((e=stream.ReadByte())>=0){
+trailing.Append(" ");
+trailing.AppendFormat("{0:x2}",(byte)e);
+}
+thrownewBadImageFormatException($"Stream has trailing data?! {{{trailing}}}");
+}
 }
 
 publicstaticboolIsClassFile(Streamstream)
@@ -213,6 +220,10 @@ public enum ClassAccessFlags {
 Synthetic=0x1000,
 Annotation=0x2000,
 Enum=0x4000,
+Module=0x8000,
+
+// This is not a real Java ClassAccessFlags, it is used to denote non-exported public types
+Internal=0x10000000,
 }
 }
 

src/Xamarin.Android.Tools.Bytecode/ClassPath.cs

@@ -88,9 +88,6 @@ static bool ShouldLoadEntry (ZipArchiveEntry entry)
 if(entry.Length==0)
 returnfalse;
 
-if(entry.Name=="module-info.class")
-returnfalse;
-
 if(entry.Name.EndsWith(".jnilib",StringComparison.OrdinalIgnoreCase))
 returnfalse;
 
@@ -111,6 +108,14 @@ public void Add (ClassFile classFile)
 classFiles.Add(classFile);
 }
 
+publicvoidAdd(ClassPathclassPath,boolremoveModules=true)
+{
+classPath.FixupModuleVisibility(removeModules);
+foreach(varcinclassPath.classFiles){
+Add(c);
+}
+}
+
 publicReadOnlyDictionary<string,List<ClassFile>>GetPackages()
 {
 returnnewReadOnlyDictionary<string,List<ClassFile>>(classFiles
@@ -360,6 +365,7 @@ public XElement ToXElement ()
 FixUpParametersFromClasses();
 
 KotlinFixups.Fixup(classFiles);
+FixupModuleVisibility(removeModules:true);
 
 varpackagesDictionary=GetPackages();
 varapi=newXElement("api",
@@ -375,6 +381,42 @@ packagesDictionary [p].OrderBy (c => c.ThisClass.Name.Value, StringComparer.Ordi
 returnapi;
 }
 
+publicvoidFixupModuleVisibility(boolremoveModules)
+{
+varpublicPackages=newHashSet<string>();
+
+varmoduleFiles=classFiles.Where(c =>c.AccessFlags==ClassAccessFlags.Module)
+.ToList();
+if(moduleFiles.Count==0){
+return;
+}
+foreach(varmoduleFileinmoduleFiles){
+if(removeModules){
+classFiles.Remove(moduleFile);
+}
+foreach(varmoduleAttrinmoduleFile.Attributes.OfType<ModuleAttribute>()){
+foreach(varexportinmoduleAttr.Exports){
+publicPackages.Add(export.Exports);
+}
+}
+}
+
+foreach(varcinclassFiles){
+if(!c.AccessFlags.HasFlag(ClassAccessFlags.Public)){
+continue;
+}
+varjniName=c.ThisClass.Name.Value;
+varpackageEnd=jniName.LastIndexOf('/');
+if(packageEnd<0){
+continue;
+}
+varpackage=jniName.Substring(0,packageEnd);
+if(!publicPackages.Contains(package)){
+c.AccessFlags=KotlinFixups.SetVisibility(c.AccessFlags,ClassAccessFlags.Internal);
+}
+}
+}
+
 publicvoidSaveXmlDescription(stringfileName)
 {
 varencoding=newUTF8Encoding(encoderShouldEmitUTF8Identifier:false);
@@ -395,5 +437,7 @@ public void SaveXmlDescription (TextWriter textWriter)
 contents.Save(writer);
 textWriter.WriteLine();
 }
+
+publicIEnumerable<ClassFile>GetClassFiles()=>classFiles;
 }
 }

src/Xamarin.Android.Tools.Bytecode/Kotlin/KotlinFixups.cs

@@ -102,7 +102,7 @@ static void FixupClassVisibility (ClassFile klass, KotlinClass metadata)
 }
 
 // Passing null for 'newVisibility' parameter means 'package-private'
-staticClassAccessFlagsSetVisibility(ClassAccessFlagsexisting,ClassAccessFlags?newVisibility)
+internalstaticClassAccessFlagsSetVisibility(ClassAccessFlagsexisting,ClassAccessFlags?newVisibility)
 {
 // First we need to remove any existing visibility flags,
 // without modifying other flags like Abstract