[Java.Interop.Tools.JavaSource] Fix tag parsing errors (#997)

Improve tag parsing by adding grammar rules for `@hide`,
`@inheritDoc`, and `{@see}`. The new grammar rules are not complete,
but these new grammar rules, along with improvements to existing
rules, reduces the number of `## Unable to translate remarks for …`
errors from 740 to 391, nearly in half.
The rules for `@param`, `@return`, `@throws`, and `@unknown` have
been updated to handle more variations of these tags. In some cases,
these tags are only followed by a single word or no content at all.
New line character filters and tab have been added to `nonSpaceTerm`
to fix instances where an "empty" tag would be merged with the tag on
the following line.
See [`Android.Database.DatabaseUtils.GetCollationKey(String)`][0] as
an example.
Before `nonSpaceTerm` change:
<param name="name">To be added.</param>
<param name="name&#xA;@return&#xA;@return">the collation key</param>
<summary>return the collation key</summary>
<returns>To be added.</returns>
After `nonSpaceTerm` change:
<param name="name">name</param>
<summary>return the collation key</summary>
<returns>the collation key</returns>
[0]: https://github.com/xamarin/android-api-docs/blob/a7e914965a0e80c9454149399551d87a18997b2e/docs/Mono.Android/en/Android.Database/DatabaseUtils.xml#L1505-L1508

Author: pjcollins

src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.BlockTagsBnfTerms.cs

@@ -25,6 +25,8 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 |DeprecatedDeclaration
 |DeprecatedSinceDeclaration
 |ExceptionDeclaration
+|InheritDocDeclaration
+|HideDeclaration
 |ParamDeclaration
 |ReturnDeclaration
 |SeeDeclaration
@@ -78,7 +80,7 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 parseNode.AstNode=p;
 };
 
-varnonSpaceTerm=newRegexBasedTerminal("[^ ]","[^ ]+"){
+varnonSpaceTerm=newRegexBasedTerminal("[^ \n\r\t]","[^ \n\r\t]+"){
 AstConfig=newAstNodeConfig{
 NodeCreator=(context,parseNode)=>parseNode.AstNode=parseNode.Token.Value,
 },
@@ -99,19 +101,42 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 FinishParse(context,parseNode);
 };
 
-ParamDeclaration.Rule="@param"+nonSpaceTerm+BlockValues;
+// Ignore @hide tags
+HideDeclaration.Rule="@hide";
+HideDeclaration.AstConfig.NodeCreator=(context,parseNode)=>{
+FinishParse(context,parseNode);
+};
+
+InheritDocDeclaration.Rule="@inheritDoc";
+InheritDocDeclaration.AstConfig.NodeCreator=(context,parseNode)=>{
+if(!grammar.ShouldImport(ImportJavadoc.InheritDocTag)){
+return;
+}
+// TODO: Iterate through parents for corresponding javadoc element.
+FinishParse(context,parseNode);
+};
+
+ParamDeclaration.Rule="@param"+nonSpaceTerm
+|"@param"+nonSpaceTerm+BlockValues
+;
 ParamDeclaration.AstConfig.NodeCreator=(context,parseNode)=>{
 if(!grammar.ShouldImport(ImportJavadoc.ParamTag)){
 return;
 }
-varp=newXElement("param",
-newXAttribute("name",string.Join("",AstNodeToXmlContent(parseNode.ChildNodes[1]))),
-AstNodeToXmlContent(parseNode.ChildNodes[2]));
+varparamName=string.Join("",AstNodeToXmlContent(parseNode.ChildNodes[1]));
+varp=newXElement("param",newXAttribute("name",paramName));
+if(parseNode.ChildNodes.Count>=3){
+p.Add(AstNodeToXmlContent(parseNode.ChildNodes[2]));
+}else{
+p.Add(paramName);
+}
 FinishParse(context,parseNode).Parameters.Add(p);
 parseNode.AstNode=p;
 };
 
-ReturnDeclaration.Rule="@return"+BlockValues;
+ReturnDeclaration.Rule="@return"
+|"@return"+BlockValues
+;
 ReturnDeclaration.AstConfig.NodeCreator=(context,parseNode)=>{
 if(!grammar.ShouldImport(ImportJavadoc.ReturnTag)){
 return;
@@ -158,7 +183,9 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 parseNode.AstNode=p;
 };
 
-ThrowsDeclaration.Rule="@throws"+nonSpaceTerm+BlockValues;
+ThrowsDeclaration.Rule="@throws"+nonSpaceTerm
+|"@throws"+nonSpaceTerm+BlockValues
+;
 ThrowsDeclaration.AstConfig.NodeCreator=(context,parseNode)=>{
 if(!grammar.ShouldImport(ImportJavadoc.ExceptionTag)){
 return;
@@ -205,7 +232,9 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 parseNode.AstNode=parseNode.Token.Value.ToString();
 
 
-UnknownTagDeclaration.Rule=unknownTagTerminal+BlockValues;
+UnknownTagDeclaration.Rule=unknownTagTerminal
+|unknownTagTerminal+BlockValues
+;
 UnknownTagDeclaration.AstConfig.NodeCreator=(context,parseNode)=>{
 if(!grammar.ShouldImport(ImportJavadoc.Remarks)){
 return;
@@ -242,6 +271,8 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 publicreadonlyNonTerminalDeprecatedDeclaration=newNonTerminal(nameof(DeprecatedDeclaration));
 publicreadonlyNonTerminalDeprecatedSinceDeclaration=newNonTerminal(nameof(DeprecatedSinceDeclaration));
 publicreadonlyNonTerminalExceptionDeclaration=newNonTerminal(nameof(ExceptionDeclaration));
+publicreadonlyNonTerminalHideDeclaration=newNonTerminal(nameof(HideDeclaration));
+publicreadonlyNonTerminalInheritDocDeclaration=newNonTerminal(nameof(InheritDocDeclaration));
 publicreadonlyNonTerminalParamDeclaration=newNonTerminal(nameof(ParamDeclaration));
 publicreadonlyNonTerminalReturnDeclaration=newNonTerminal(nameof(ReturnDeclaration));
 publicreadonlyNonTerminalSeeDeclaration=newNonTerminal(nameof(SeeDeclaration));

src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.InlineTagsBnfTerms.cs

@@ -26,6 +26,7 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 |LinkDeclaration
 |LinkplainDeclaration
 |LiteralDeclaration
+|SeeDeclaration
 |ValueDeclaration
 ;
 
@@ -75,6 +76,14 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 parseNode.AstNode=newXText(content);
 };
 
+SeeDeclaration.Rule=grammar.ToTerm("{@see")+InlineValue+"}";
+SeeDeclaration.AstConfig.NodeCreator=(context,parseNode)=>{
+// TODO: @see supports multiple forms; see: https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#see
+// Also need to convert to appropriate CREF value, ignore for now
+vartarget=parseNode.ChildNodes[1].AstNode;
+parseNode.AstNode=newXElement("c",target);
+};
+
 ValueDeclaration.Rule=grammar.ToTerm("{@value}")
 |grammar.ToTerm("{@value")+InlineValue+"}";
 ValueDeclaration.AstConfig.NodeCreator=(context,parseNode)=>{
@@ -114,10 +123,12 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 publicreadonlyNonTerminalLinkplainDeclaration=newNonTerminal(nameof(LinkplainDeclaration));
 
 // https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#literal
-publicreadonlyNonTerminalLiteralDeclaration=newNonTerminal(nameof(LinkplainDeclaration));
+publicreadonlyNonTerminalLiteralDeclaration=newNonTerminal(nameof(LiteralDeclaration));
+
+publicreadonlyNonTerminalSeeDeclaration=newNonTerminal(nameof(SeeDeclaration));
 
 // https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#value
-publicreadonlyNonTerminalValueDeclaration=newNonTerminal(nameof(ValueDeclaration));
+publicreadonlyNonTerminalValueDeclaration=newNonTerminal(nameof(ValueDeclaration));
 }
 }
 }

src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocParser.cs

@@ -26,6 +26,7 @@ internal enum ImportJavadoc {
 SinceTag=1<<9,
 VersionTag=1<<10,
 ExtraRemarks=1<<11,
+InheritDocTag=1<<12,
 }
 
 [Flags]
@@ -43,6 +44,7 @@ public enum XmldocStyle {
 |ImportJavadoc.SinceTag
 |ImportJavadoc.VersionTag
 |ImportJavadoc.ExtraRemarks
+|ImportJavadoc.InheritDocTag
 ,
 IntelliSense=ImportJavadoc.Summary
 |ImportJavadoc.ExceptionTag

tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.BlockTagsBnfTermsTests.cs

@@ -56,14 +56,39 @@ public void ExceptionDeclaration ()
 //Assert.AreEqual ("<exception cref=\"Throwable\">Just Because.</exception>", r.Root.AstNode.ToString ());
 }
 
+[Test]
+publicvoidInheritDocDeclaration()
+{
+varp=CreateParser(g =>g.BlockTagsTerms.InheritDocDeclaration);
+
+varr=p.Parse("@inheritDoc");
+Assert.IsFalse(r.HasErrors(),"@inheritDoc: "+DumpMessages(r,p));
+// TODO: Enable after adding support for @inheritDoc
+Assert.IsNull(r.Root.AstNode,"@inheritDoc should be ignored, but node was not null.");
+}
+
+[Test]
+publicvoidHideDeclaration()
+{
+varp=CreateParser(g =>g.BlockTagsTerms.HideDeclaration);
+
+varr=p.Parse("@hide");
+Assert.IsFalse(r.HasErrors(),"@hide: "+DumpMessages(r,p));
+Assert.IsNull(r.Root.AstNode,"@hide should be ignored, but node was not null.");
+}
+
 [Test]
 publicvoidParamDeclaration()
 {
 varp=CreateParser(g =>g.BlockTagsTerms.ParamDeclaration);
 
-varr=p.Parse("@param a Insert description here\n");
+varr=p.Parse("@param a Insert description here\nand here.");
 Assert.IsFalse(r.HasErrors(),"@param: "+DumpMessages(r,p));
-Assert.AreEqual("<param name=\"a\">Insert description here</param>",r.Root.AstNode.ToString());
+Assert.AreEqual($"<param name=\"a\">Insert description here{Environment.NewLine}and here.</param>",r.Root.AstNode.ToString());
+
+r=p.Parse("@param b");
+Assert.IsFalse(r.HasErrors(),"name only @param: "+DumpMessages(r,p));
+Assert.AreEqual("<param name=\"b\">b</param>",r.Root.AstNode.ToString());
 }
 
 [Test]
@@ -120,16 +145,21 @@ public void ThrowsDeclaration ()
 varp=CreateParser(g =>g.BlockTagsTerms.ThrowsDeclaration);
 
 varr=p.Parse("@throws Throwable the {@code Exception} raised by this method");
-Assert.IsFalse(r.HasErrors(),"@throws: "+DumpMessages(r,p));
+Assert.IsFalse(r.HasErrors(),"{@code} @throws: "+DumpMessages(r,p));
 Assert.IsNull(r.Root.AstNode,"@throws should be ignored, but node with code block was not null.");
 // TODO: Re-enable when we can generate valid crefs
 //Assert.AreEqual ("<exception cref=\"Throwable\">the <c>Exception</c> raised by this method</exception>", r.Root.AstNode.ToString ());
 
 r=p.Parse("@throws Throwable something <i>or other</i>!");
-Assert.IsFalse(r.HasErrors(),"@throws: "+DumpMessages(r,p));
-Assert.IsNull(r.Root.AstNode,"@throws should be ignored, but node was not null.");
+Assert.IsFalse(r.HasErrors(),"<i> @throws: "+DumpMessages(r,p));
+Assert.IsNull(r.Root.AstNode,"@throws should be ignored, but node with <i> was not null.");
 // TODO: Re-enable when we can generate valid crefs
 //Assert.AreEqual ("<exception cref=\"Throwable\">something <i>or other</i>!</exception>", r.Root.AstNode.ToString ());
+
+r=p.Parse("@throws android.content.ActivityNotFoundException");
+Assert.IsFalse(r.HasErrors(),"name only @throws: "+DumpMessages(r,p));
+// TODO: Re-enable when we can generate valid crefs
+Assert.IsNull(r.Root.AstNode,"@throws should be ignored, but name only node was not null.");
 }
 
 [Test]
@@ -140,6 +170,10 @@ public void UnknownTagDeclaration ()
 varr=p.Parse("@this-is-not-supported something {@code foo} else.");
 Assert.IsFalse(r.HasErrors(),"@this-is-not-supported: "+DumpMessages(r,p));
 Assert.AreEqual(null,r.Root.AstNode);
+
+r=p.Parse("@standalonetag");
+Assert.IsFalse(r.HasErrors(),"@standalonetag: "+DumpMessages(r,p));
+Assert.AreEqual(null,r.Root.AstNode);
 }
 }
 }

tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.InlineTagsBnfTermsTests.cs

@@ -75,6 +75,16 @@ public void LiteralDeclaration ()
 Assert.AreEqual("A<B>C",r.Root.AstNode.ToString());
 }
 
+[Test]
+publicvoidSeeDeclaration()
+{
+varp=CreateParser(g =>g.InlineTagsTerms.SeeDeclaration);
+
+varr=p.Parse("{@see #cancelNotification(String, String, int)}");
+Assert.IsFalse(r.HasErrors(),DumpMessages(r,p));
+Assert.AreEqual("<c>#cancelNotification(String, String, int)</c>",r.Root.AstNode.ToString());
+}
+
 [Test]
 publicvoidValueDeclaration()
 {

tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs

@@ -104,11 +104,15 @@ What about soft paragraphs?
 <p>What about <i>hard</i> paragraphs?
 
 @param a something
+@param b
+@param c
 @see #method()
 @apiSince 1
 ",
 FullXml=@"<member>
  <param name=""a"">something</param>
+ <param name=""b"">b</param>
+ <param name=""c"">c</param>
  <summary>This is the summary sentence.</summary>
  <remarks>
  <para>This is the summary sentence. Insert
@@ -120,6 +124,8 @@ more description here.</para>
 </member>",
 IntelliSenseXml=@"<member>
  <param name=""a"">something</param>
+ <param name=""b"">b</param>
+ <param name=""c"">c</param>
  <summary>This is the summary sentence.</summary>
 </member>",
 },
@@ -183,19 +189,25 @@ more description here.
 @param manifest The value of the <a
 href=""{@docRoot}guide/topics/manifest/manifest-element.html#vcode"">{@code
 android:versionCode}</a> manifest attribute.
+@param empty
+@return the return value
 ",
 FullXml=$@"<member>
  <param name=""manifest"">The value of the <see href=""{DocRootPrefixExpected}guide/topics/manifest/manifest-element.html#vcode""><c>android:versionCode</c></see> manifest attribute.</param>
+ <param name=""empty"">empty</param>
  <summary>See <see href=""http://man7.org/linux/man-pages/man2/accept.2.html"">accept(2)</see>.</summary>
  <remarks>
  <para>See <see href=""http://man7.org/linux/man-pages/man2/accept.2.html"">accept(2)</see>. Insert
 more description here.
 How about another link <see href=""http://man7.org/linux/man-pages/man2/accept.2.html"">accept(2)</see></para>
  </remarks>
+ <returns>the return value</returns>
 </member>",
 IntelliSenseXml=$@"<member>
  <param name=""manifest"">The value of the <see href=""{DocRootPrefixExpected}guide/topics/manifest/manifest-element.html#vcode""><c>android:versionCode</c></see> manifest attribute.</param>
+ <param name=""empty"">empty</param>
  <summary>See <see href=""http://man7.org/linux/man-pages/man2/accept.2.html"">accept(2)</see>.</summary>
+ <returns>the return value</returns>
 </member>",
 },
 };