Skip to content

Commit befaed4

Browse files
jonpryorjonpryor
committed
[generator] Add generator --with-javadoc-xml=FILE support.
Commit 69e1b80 added `tools/java-source-utils`, which along with b588ef5, can parse Java source code and extract Javadoc comments from Android API-30 sources into an XML file: into an XML file: $ find $HOME/android-toolchain/sdk/platforms/android-30/src/{android,java,javax,org} -iname \*.java \ | grep -v '\.annotated\.' > sources.txt $ java -jar java-source-utils.jar -v \ --source "$HOME/android-toolchain/sdk/platforms/android-30/src" \ --output-javadoc android-javadoc.xml \ @sources.txt What can we *do* with the generated `android-javadoc.xml`? `android-javadoc.xml` contains parameter names, and thus can be used with `class-parse --docspath`; see commit 806082f. What we *really* want to do is make the Javadoc information *useful* to consumers of the binding assembly. This means that we want a C# XML Documentation file for the binding assembly. The most straightforward way to get a C# XML Documentation File is to emit [C# XML Documentation Comments][0] into the binding source code! Add a new `generator --with-javadoc-xml=FILE` option. When specified, `FILE` will be treated as an XML file containing the output from `java-source-utils.jar --output-javadoc` (69e1b80), and all `<javadoc/>` elements within the XML file will be associated with C# types and members to emit, based on the `//@jni-signature` and `//@name` attributes, as appropriate. When the bindings are written to disk, the Javadoc comments will be translated to C# XML Documentation comments, in a "best effort" basis. (THIS WILL BE INCOMPLETE.) To perform the Javadoc-to-C# XML Documentation comments conversion, add a new Irony-based grammar to `Java.Interop.Tools.JavaSource.dll`, in the new `Java.Interop.Tools.JavaSource.SourceJavadocToXmldocParser` type, which parses the Javadoc content and translates to XML. In addition to transforming the Javadoc comments into C# XML documentation comments, we *also* want to provide "upstream information" in the form of: 1. A URL to the corresponding online Javadoc HTML documentation. 2. A copyright notice disclaimer. Allow provision of this information by updating `java-source-utils.jar` to support the new options: --doc-copyright FILE Copyright information for Javadoc. Should be in mdoc(5) XML, to be held within <remarks/>. Stored in //javadoc-metadata/copyright. --doc-url-prefix URL Base URL for links to documentation. Stored in //javadoc-metadata/link/@Prefix. --doc-url-style STYLE STYLE of URLs to generate for member links. Stored in //javadoc-metadata/link/@Style. Supported styles include: - developer.android.com/reference@2020-Nov The new `/api/javadoc-metadata/link@prefix` and `/api/javadoc-metadata/link@style` XML attributes, stored within `java-source-utils.jar --output-javadoc` XML output, allow construction of a URL to the Java member. For example, given: java -jar java-source-utils.jar \ --doc-url-prefix https://developer.android.com/reference \ --doc-url-style developer.android.com/reference@2020-Nov Then `generator` can emit the C# documentation comment for `java.lang.Object.equals(Object)`: /// <format type="text/html"> /// <a href="https://developer.android.com/reference/java/lang/Object#equals(java.lang.Object)">Java documentation for <tt>java.lang.Object.equals(java.lang.Object)</tt>.</a> /// </format> The copyright notice disclaimer is supported by `java-source-utils.jar --doc-copyright FILE`; the contents of `FILE` are inserted into the `/api/javadoc-metadata/copyright` element, and will be copied into the output of every C# XML documentation block. Example output is at: * <https://gist.github.com/jonpryor/ee60a7385e908458bbe9ad48e30b2ecc> TODO: * Fix `java-source-utils.jar` & `//*/@jni-signature` so that [generics types are properly represented][1]. * Properties don't yet emit C# XML documentation comments, e.g. `Java.Lang.Object.Class` has no documentation, even though `java.lang.Object.getClass()` is documented. * `SourceJavadocToXmldocParser` doesn't support many constructs. [0]: https://docs.microsoft.com/en-us/dotnet/csharp/codedoc [1]: #687 (comment)
1 parent 99897b2 commit befaed4

36 files changed

+2070
-22
lines changed

‎src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/IronyExtensions.cs

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Linq;
5+
using System.Text;
6+
using Irony.Ast;
7+
using Irony.Parsing;
8+
9+
namespace Java.Interop.Tools.JavaSource {
10+
11+
static class IronyExtensions {
12+
13+
public static void MakePlusRule (this NonTerminal star, Grammar grammar, BnfTerm delimiter)
14+
{
15+
star.Rule = grammar.MakePlusRule (star, delimiter);
16+
}
17+
18+
public static void MakeStarRule (this NonTerminal star, Grammar grammar, BnfTerm delimiter, BnfTerm of)
19+
{
20+
star.Rule = grammar.MakeStarRule (star, delimiter, of);
21+
}
22+
23+
public static void MakeStarRule (this NonTerminal star, Grammar grammar, BnfTerm of)
24+
{
25+
star.Rule = grammar.MakeStarRule (star, of);
26+
}
27+
}
28+
}

‎src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/JavadocInfo.cs

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Collections.ObjectModel;
4+
using System.IO;
5+
using System.Linq;
6+
using System.Xml.Linq;
7+
8+
using Irony.Ast;
9+
using Irony.Parsing;
10+
11+
namespace Java.Interop.Tools.JavaSource {
12+
13+
sealed class JavadocInfo {
14+
public readonly ICollection<XNode> Exceptions = new Collection<XNode> ();
15+
public readonly ICollection<XNode> Extra = new Collection<XNode> ();
16+
public readonly ICollection<XNode> Remarks = new Collection<XNode> ();
17+
public readonly ICollection<XNode> Parameters = new Collection<XNode> ();
18+
public readonly ICollection<XNode> Returns = new Collection<XNode> ();
19+
20+
public override string ToString ()
21+
{
22+
return new XElement ("Javadoc",
23+
new XElement (nameof (Parameters), Parameters),
24+
new XElement (nameof (Remarks), Remarks),
25+
new XElement (nameof (Returns), Returns),
26+
new XElement (nameof (Exceptions), Exceptions),
27+
new XElement (nameof (Extra), Extra))
28+
.ToString ();
29+
}
30+
}
31+
}

‎src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.BlockTagsBnfTerms.cs

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Linq;
5+
using System.Xml.Linq;
6+
7+
using Irony.Ast;
8+
using Irony.Parsing;
9+
10+
namespace Java.Interop.Tools.JavaSource {
11+
12+
public partial class SourceJavadocToXmldocGrammar {
13+
14+
// https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#javadoctags
15+
public class BlockTagsBnfTerms {
16+
17+
internal BlockTagsBnfTerms ()
18+
{
19+
}
20+
21+
internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
22+
{
23+
AllBlockTerms.Rule = AuthorDeclaration
24+
| ApiSinceDeclaration
25+
| DeprecatedDeclaration
26+
| DeprecatedSinceDeclaration
27+
| ExceptionDeclaration
28+
| ParamDeclaration
29+
| ReturnDeclaration
30+
| SeeDeclaration
31+
| SerialDataDeclaration
32+
| SerialFieldDeclaration
33+
| SinceDeclaration
34+
| ThrowsDeclaration
35+
| UnknownTagDeclaration
36+
| VersionDeclaration
37+
;
38+
BlockValue.Rule = grammar.HtmlTerms.ParsedCharacterData
39+
| grammar.HtmlTerms.InlineDeclaration
40+
;
41+
BlockValues.MakePlusRule (grammar, BlockValue);
42+
43+
AuthorDeclaration.Rule = "@author" + BlockValues;
44+
AuthorDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
45+
// Ignore; not sure how best to convert to Xmldoc
46+
FinishParse (context, parseNode);
47+
};
48+
49+
ApiSinceDeclaration.Rule = "@apiSince" + BlockValues;
50+
ApiSinceDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
51+
var p = new XElement ("para", "Added in API level ", AstNodeToXmlContent (parseNode.ChildNodes [1]), ".");
52+
FinishParse (context, parseNode).Remarks.Add (p);
53+
parseNode.AstNode = p;
54+
};
55+
56+
DeprecatedDeclaration.Rule = "@deprecated" + BlockValues;
57+
DeprecatedDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
58+
var p = new XElement ("para", "This member is deprecated. ", AstNodeToXmlContent (parseNode.ChildNodes [1]));
59+
FinishParse (context, parseNode).Remarks.Add (p);
60+
parseNode.AstNode = p;
61+
};
62+
63+
DeprecatedSinceDeclaration.Rule = "@deprecatedSince" + BlockValues;
64+
DeprecatedSinceDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
65+
var p = new XElement ("para", "This member was deprecated in API level ", AstNodeToXmlContent (parseNode.ChildNodes [1]), ".");
66+
FinishParse (context, parseNode).Remarks.Add (p);
67+
parseNode.AstNode = p;
68+
};
69+
70+
var nonSpaceTerm = new RegexBasedTerminal ("[^ ]", "[^ ]+") {
71+
AstConfig = new AstNodeConfig {
72+
NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value,
73+
},
74+
};
75+
76+
ExceptionDeclaration.Rule = "@exception" + nonSpaceTerm + BlockValues;
77+
ExceptionDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
78+
// TODO: convert `nonSpaceTerm` into a proper CREF
79+
var e = new XElement ("exception",
80+
new XAttribute ("cref", string.Join ("", AstNodeToXmlContent (parseNode.ChildNodes [1]))),
81+
AstNodeToXmlContent (parseNode.ChildNodes [2]));
82+
FinishParse (context, parseNode).Exceptions.Add (e);
83+
parseNode.AstNode = e;
84+
};
85+
86+
ParamDeclaration.Rule = "@param" + nonSpaceTerm + BlockValues;
87+
ParamDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
88+
var p = new XElement ("param",
89+
new XAttribute ("name", string.Join ("", AstNodeToXmlContent (parseNode.ChildNodes [1]))),
90+
AstNodeToXmlContent (parseNode.ChildNodes [2]));
91+
FinishParse (context, parseNode).Parameters.Add (p);
92+
parseNode.AstNode = p;
93+
};
94+
95+
ReturnDeclaration.Rule = "@return" + BlockValues;
96+
// ReturnDeclaration.Flags = TermFlags.IsMultiline;
97+
ReturnDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
98+
var r = new XElement ("returns",
99+
AstNodeToXmlContent (parseNode.ChildNodes [1]));
100+
FinishParse (context, parseNode).Returns.Add (r);
101+
parseNode.AstNode = r;
102+
};
103+
104+
SeeDeclaration.Rule = "@see" + BlockValues;
105+
SeeDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
106+
// TODO: @see supports multiple forms; see: https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#see
107+
var e = new XElement ("altmember",
108+
new XAttribute ("cref", string.Join ("", AstNodeToXmlContent (parseNode.ChildNodes [1]))));
109+
FinishParse (context, parseNode).Extra.Add (e);
110+
parseNode.AstNode = e;
111+
};
112+
113+
SinceDeclaration.Rule = "@since" + BlockValues;
114+
SinceDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
115+
var p = new XElement ("para", "Added in ", AstNodeToXmlContent (parseNode.ChildNodes [1]), ".");
116+
FinishParse (context, parseNode).Remarks.Add (p);
117+
parseNode.AstNode = p;
118+
};
119+
120+
ThrowsDeclaration.Rule = "@throws" + nonSpaceTerm + BlockValues;
121+
ThrowsDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
122+
// TODO: convert `nonSpaceTerm` into a proper CREF
123+
var e = new XElement ("exception",
124+
new XAttribute ("cref", string.Join ("", AstNodeToXmlContent (parseNode.ChildNodes [1]))),
125+
AstNodeToXmlContent (parseNode.ChildNodes [2]));
126+
FinishParse (context, parseNode).Exceptions.Add (e);
127+
parseNode.AstNode = e;
128+
};
129+
130+
// Ignore serialization informatino
131+
SerialDeclaration.Rule = "@serial" + BlockValues;
132+
SerialDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
133+
FinishParse (context, parseNode);
134+
};
135+
136+
SerialDataDeclaration.Rule = "@serialData" + BlockValues;
137+
SerialDataDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
138+
FinishParse (context, parseNode);
139+
};
140+
141+
SerialFieldDeclaration.Rule = "@serialField" + BlockValues;
142+
SerialFieldDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
143+
FinishParse (context, parseNode);
144+
};
145+
146+
var unknownTagTerminal = new RegexBasedTerminal ("@[unknown]", @"@\S+") {
147+
Priority = TerminalPriority.Low,
148+
};
149+
unknownTagTerminal.AstConfig.NodeCreator = (context, parseNode) =>
150+
parseNode.AstNode = parseNode.Token.Value.ToString ();
151+
152+
153+
UnknownTagDeclaration.Rule = unknownTagTerminal + BlockValues;
154+
UnknownTagDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
155+
Console.WriteLine ($"# Unsupported @block-tag value: {parseNode.ChildNodes [0].AstNode}");
156+
FinishParse (context, parseNode);
157+
};
158+
159+
// Ignore Version
160+
VersionDeclaration.Rule = "@version" + BlockValues;
161+
VersionDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
162+
FinishParse (context, parseNode);
163+
};
164+
}
165+
166+
public readonly NonTerminal AllBlockTerms = new NonTerminal (nameof (AllBlockTerms), ConcatChildNodes);
167+
168+
public readonly Terminal Cdata = new CharacterDataTerminal ("#CDATA", preserveLeadingWhitespace: true);
169+
/*
170+
public readonly Terminal Cdata = new RegexBasedTerminal (nameof (BlockValue), "[^<]*") {
171+
AstConfig = new AstNodeConfig {
172+
NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value.ToString (),
173+
},
174+
};
175+
*/
176+
177+
public readonly NonTerminal BlockValue = new NonTerminal (nameof (BlockValue), ConcatChildNodes);
178+
public readonly NonTerminal BlockValues = new NonTerminal (nameof (BlockValues), ConcatChildNodes);
179+
public readonly NonTerminal AuthorDeclaration = new NonTerminal (nameof (AuthorDeclaration));
180+
public readonly NonTerminal ApiSinceDeclaration = new NonTerminal (nameof (ApiSinceDeclaration));
181+
public readonly NonTerminal DeprecatedDeclaration = new NonTerminal (nameof (DeprecatedDeclaration));
182+
public readonly NonTerminal DeprecatedSinceDeclaration = new NonTerminal (nameof (DeprecatedSinceDeclaration));
183+
public readonly NonTerminal ExceptionDeclaration = new NonTerminal (nameof (ExceptionDeclaration));
184+
public readonly NonTerminal ParamDeclaration = new NonTerminal (nameof (ParamDeclaration));
185+
public readonly NonTerminal ReturnDeclaration = new NonTerminal (nameof (ReturnDeclaration));
186+
public readonly NonTerminal SeeDeclaration = new NonTerminal (nameof (SeeDeclaration));
187+
public readonly NonTerminal SerialDeclaration = new NonTerminal (nameof (SerialDeclaration));
188+
public readonly NonTerminal SerialDataDeclaration = new NonTerminal (nameof (SerialDataDeclaration));
189+
public readonly NonTerminal SerialFieldDeclaration = new NonTerminal (nameof (SerialFieldDeclaration));
190+
public readonly NonTerminal SinceDeclaration = new NonTerminal (nameof (SinceDeclaration));
191+
public readonly NonTerminal ThrowsDeclaration = new NonTerminal (nameof (ThrowsDeclaration));
192+
public readonly NonTerminal UnknownTagDeclaration = new NonTerminal (nameof (UnknownTagDeclaration));
193+
public readonly NonTerminal VersionDeclaration = new NonTerminal (nameof (VersionDeclaration));
194+
}
195+
}
196+
}

0 commit comments

Comments
 (0)