[build] Fix for Issue 3232 (antlr#3237)

* Some changes for fixing antlr#3232

* Fix datalog grammar and example.

* Update to latest trgen and desc.xml. Add desc.xml for sql/athena, fix grammar to work on TypeScript.

* Add sql/hive/v4 to testing.

* Update trgen. Fix mysql so that Oracle version can be added.

* Update script to assume Trash already installed. Bump version of Trash to 0.20.9. Add EOF to start rules for Ada grammars so discovery in trgen works.

* Disambiguate start rule for powerquery. Fix differences in newlines across OSes.

* Fix testing of JavaScript if only testing Java. Update to latest trgen that has os-targets elements in desc.xml, in order to limit problem OSes. Fix javascript/ecmascript--Go target does not work.

* Use trgen 0.20.11, which corrects a small problem in additional targets even if one specified. Add in 4.7 requirement for unicode extended and unicode properties. Fix test.ps1 "start2" issue in builds.

.github/workflows/main.yml

@@ -97,7 +97,7 @@ jobs:
     - name: Install trgen
       shell: bash
       run: |
-        for i in trgen triconv trwdog trxml trxml2; do dotnet tool install -g $i --version 0.20.6; done
+        for i in trgen triconv trwdog trxml trxml2; do dotnet tool install -g $i --version 0.20.11; done
     - name: Test
       shell: pwsh
       run: |
@@ -197,7 +197,7 @@ jobs:
     - name: Install trgen
       shell: bash
       run: |
-        for i in trgen triconv trwdog trxml trxml2; do dotnet tool install -g $i --version 0.20.6; done
+        for i in trgen triconv trwdog trxml trxml2; do dotnet tool install -g $i --version 0.20.11; done
     - name: Test
       shell: bash
       run: |

_scripts/gen-desc.sh

@@ -0,0 +1,190 @@
+#!/usr/bin/bash
+
+# true or false
+quiet=true
+
+# Get full path of this script, and root.
+full_path_script=$(realpath $0)
+full_path_script_dir=`dirname $full_path_script`
+full_path_templates=$(dirname $full_path_script)/templates
+pushd $full_path_script_dir
+cd ..
+root=`pwd`
+popd
+
+# Sanity checks for required environment.
+unameOut="$(uname -s)"
+case "${unameOut}" in
+    Linux*)     machine=Linux;;
+    Darwin*)    machine=Mac;;
+    CYGWIN*)    machine=Cygwin;;
+    MINGW*)     machine=MinGw;;
+    *)          machine="UNKNOWN:${unameOut}"
+esac
+if [[ "$machine" != "Linux" && "$machine" != "Mac" && "$machine" != "MinGw" ]]
+then
+    echo "This script runs only in a Linux environment. Skipping."
+    exit 0
+fi
+
+# string contains(List<string> list, string item)
+# If "list" contains the "item", return "yes" else return "no".
+contains() {
+    if [[ " $1 " =~ " $2 " ]]
+    then
+        echo yes
+    else
+        echo no
+    fi
+}
+
+# List<string> add(List<string> list, string name), without duplicates.
+# This function returns a new list by adding "name" to "list".
+add() {
+    if [[ `contains "$1" "$2"` == "no" ]]
+    then
+        echo "$1 $2"
+    else
+        echo "$1"
+    fi
+}
+
+setupdeps()
+{
+    trgen --version
+    if [ $? != "0" ]
+    then
+        echo "Need to set up Trash."
+        exit 0
+    fi
+}
+
+thetime()
+{
+    local hh
+    local DIFF
+    DIFF=$1
+    hh="$(($DIFF / 3600 ))"
+    mm="$((($DIFF % 3600) / 60))"
+    ss="$(($DIFF % 60))"
+    printf "%02d:%02d:%02d" $hh $mm $ss
+}
+
+# strategy="actions" or "git"
+strategy=""
+failed=""
+succeeded=""
+skipped=""
+grammars=""
+targets=""
+tests=""
+rm -rf `find . -name 'Generated*' -type d`
+
+# Parse args, and update computation to perform.
+order="grammars"
+while getopts 'g:t:h' opt; do
+  case "$opt" in
+    g)
+      order="grammars"
+      arg="$OPTARG"
+      for a in $arg
+      do
+          grammars=`add "$grammars" "$a"`
+      done
+      ;;
+
+    t)
+      order="targets"
+      arg="$OPTARG"
+      for a in $arg
+      do
+          targets=`add "$targets" "$a"`
+      done
+      ;;
+
+    ?|h)
+      echo "Usage: $(basename $0) [-a] [-b] [-c arg]"
+      exit 1
+      ;;
+  esac
+done
+
+if [ "$grammars" == "" ]
+then
+    descs=`find . -name desc.xml`
+    for i in $descs
+    do
+        echo "i = $i"
+        d=`dirname $i`
+        pushd $d > /dev/null
+        d=`pwd`
+        g=${d##*$root/}
+        testname=$g
+        grammars=`add "$grammars" "$testname"`
+        popd > /dev/null
+    done
+fi
+
+echo "grammars = $grammars"
+echo "Evaluating targets for each grammar..."
+
+targets="Antlr4cs CSharp Cpp Dart Go Java JavaScript PHP Python3 TypeScript"
+for g in $grammars
+do
+    echo Starting analysis of $g
+    cd $root/$g
+    desc=""
+    for t in $targets
+    do
+        cd $root/$g
+        rm -rf Generated-*
+        echo "$g,$t:"
+        trgen -t "$t" --template-sources-directory "$full_path_templates"
+        if [ "$?" -ne 0 ]
+        then
+            failed=`add "$failed" "$g/$t"`
+            continue
+        fi
+        for j in Generated-*
+        do
+            cd $root/$g/$j
+            echo "Build"
+            date1=$(date +"%s")
+            bash build.sh
+            status="$?"
+            date2=$(date +"%s")
+            DIFF=$(($date2-$date1))
+            length=`thetime $DIFF`
+            echo "$length "
+            if [ "$status" -ne 0 ]
+            then
+                echo Fail.
+                continue
+            fi
+            echo "Test "
+            date1=$(date +"%s")
+            bash test.sh
+            status="$?"
+            date2=$(date +"%s")
+            DIFF=$(($date2-$date1))
+            length=`thetime $DIFF`
+            echo "$length "
+            if [ "$status" != "0" ]
+            then
+                echo Fail.
+                continue
+            fi
+            echo Success.
+            if [ "$desc" == "" ]; then desc="$t"; else desc="$desc;$t"; fi
+        done
+        rm -rf Generated-*
+    done
+    echo Target description for $g is $desc
+	cd $root/$g
+    cat - <<EOF > desc2.xml
+<desc>
+   <targets>$desc</targets>
+</desc>
+EOF
+	cat desc2.xml
+done

_scripts/test.ps1

@@ -144,7 +144,7 @@ function Test-Grammar {
         }
     }
 
-    Write-Host "Test completed, time: $((Get-Date) - $start2)" -ForegroundColor Yellow
+    Write-Host "Test completed, time: $((Get-Date) - $start)" -ForegroundColor Yellow
     Set-Location $cwd
     return @{
         Success     = $success

_scripts/test.sh

@@ -36,27 +36,6 @@ then
     exit 0
 fi
 
-setupdeps()
-{
-    trgen --version
-    if [ $? != "0" ]
-    then
-        echo "Setting up trgen and antlr jar."
-        dotnet tool install -g trgen --version 0.20.6
-        dotnet tool install -g triconv --version 0.20.6
-        dotnet tool install -g trxml2 --version 0.20.6
-        dotnet tool install -g trwdog --version 0.20.6
-    case "${unameOut}" in
-        Linux*)     curl 'https://repo1.maven.org/maven2/org/antlr/antlr4/4.11.1/antlr4-4.12.0-complete.jar' -o $antlr4jar ;;
-        Darwin*)    curl 'https://repo1.maven.org/maven2/org/antlr/antlr4/4.11.1/antlr4-4.12.0-complete.jar' -o $antlr4jar ;;
-        CYGWIN*)    curl 'https://repo1.maven.org/maven2/org/antlr/antlr4/4.11.1/antlr4-4.12.0-complete.jar' -o $antlr4jar ;;
-        MINGW*)     curl 'https://repo1.maven.org/maven2/org/antlr/antlr4/4.11.1/antlr4-4.12.0-complete.jar' -o $antlr4jar ;;
-        *)          echo 'unknown machine'
-    esac
-        echo "Done setting up."
-    fi
-}
-
 thetime()
 {
     local hh
@@ -372,8 +351,9 @@ do
     bad=`trgen -t "$target" --template-sources-directory "$full_path_templates" --antlr-tool-path $antlr4jar 2> /dev/null`
     for i in $bad; do failed+=( "$testname/$target" ); done
 
-    for d in Generated-$target*
+    for d in `echo Generated-$target-* Generated-$target`
     do
+	if [ ! -d $d ]; then continue; fi
         if [ ! -f $d/build.sh ]; then echo " no build.sh"; continue; fi
 
         # Build driver code.

abb/desc.xml

@@ -1,4 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <desc xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../_scripts/desc.xsd">
-   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;TypeScript;PHP;Python3</targets>
+   <antlr-version>^4.10</antlr-version>
+   <targets>CSharp;Cpp;Dart;Go;Java;JavaScript;PHP;Python3;TypeScript</targets>
 </desc>

abnf/desc.xml

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <desc xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../_scripts/desc.xsd">
-   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;TypeScript;PHP;Python3</targets>
+   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;PHP;Python3;TypeScript</targets>
 </desc>

acme/desc.xml

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <desc xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../_scripts/desc.xsd">
-   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;TypeScript;Python3</targets>
+   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;Python3;TypeScript</targets>
 </desc>

ada/ada2005/Ada2005Parser.g4

@@ -1032,7 +1032,7 @@ abort_statement
 
 
 compilation
-   : compilation_unit*
+   : compilation_unit* EOF
    ;
 
 compilation_unit

ada/ada2005/desc.xml

@@ -1,4 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <desc xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../../_scripts/desc.xsd">
+   <antlr-version>^4.10</antlr-version>
    <targets>CSharp;Cpp;Dart;Go;Java;JavaScript;TypeScript;PHP;Python3</targets>
 </desc>

ada/ada2012/desc.xml

@@ -1,4 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <desc xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../../_scripts/desc.xsd">
-   <targets>CSharp;Cpp;Dart;Go;Java;JavaScript;TypeScript;PHP;Python3</targets>
+   <antlr-version>^4.10</antlr-version>
+   <targets>CSharp;Cpp;Dart;Go;Java;JavaScript;PHP;Python3;TypeScript</targets>
 </desc>

ada/ada83/Ada83Parser.g4

@@ -686,7 +686,7 @@ abort_statement
 
 
 compilation
-   : compilation_unit*
+   : compilation_unit* EOF
    ;
 
 compilation_unit

ada/ada83/desc.xml

@@ -1,4 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <desc xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../../_scripts/desc.xsd">
+   <antlr-version>^4.10</antlr-version>
    <targets>CSharp;Cpp;Dart;Go;Java;JavaScript;TypeScript;PHP;Python3</targets>
 </desc>

ada/ada95/Ada95Parser.g4

@@ -975,7 +975,7 @@ abort_statement
 
 
 compilation
-   : compilation_unit*
+   : compilation_unit* EOF
    ;
 
 compilation_unit

ada/ada95/desc.xml

@@ -1,4 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <desc xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../../_scripts/desc.xsd">
+   <antlr-version>^4.10</antlr-version>
    <targets>CSharp;Cpp;Dart;Go;Java;JavaScript;TypeScript;PHP;Python3</targets>
 </desc>

agc/desc.xml

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <desc xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../_scripts/desc.xsd">
-   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;TypeScript;PHP;Python3</targets>
+   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;PHP;Python3;TypeScript</targets>
 </desc>

alef/desc.xml

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <desc xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../_scripts/desc.xsd">
-   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;TypeScript;PHP;Python3</targets>
+   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;PHP;Python3;TypeScript</targets>
 </desc>

algol60/desc.xml

@@ -1,4 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <desc xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../_scripts/desc.xsd">
-   <targets>CSharp;Cpp;Dart;Java;JavaScript;TypeScript;Python3</targets>
+   <antlr-version>^4.10</antlr-version>
+   <targets>CSharp;Cpp;Dart;Go;Java;JavaScript;Python3;TypeScript</targets>
 </desc>

alloy/desc.xml

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <desc xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../_scripts/desc.xsd">
-   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;TypeScript;PHP;Python3</targets>
+   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;PHP;Python3;TypeScript</targets>
 </desc>

alpaca/desc.xml

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <desc xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../_scripts/desc.xsd">
-   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;TypeScript;PHP;Python3</targets>
+   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;PHP;Python3;TypeScript</targets>
 </desc>

angelscript/desc.xml

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <desc xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../_scripts/desc.xsd">
-   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;TypeScript;PHP;Python3</targets>
+   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;PHP;Python3;TypeScript</targets>
 </desc>

apt/desc.xml

@@ -1,4 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <desc xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../_scripts/desc.xsd">
-   <targets>CSharp;Cpp;Dart;Go;Java;JavaScript;TypeScript;PHP;Python3</targets>
+   <antlr-version>^4.7</antlr-version>
+   <targets>CSharp;Cpp;Dart;Go;Java;JavaScript;PHP;Python3;TypeScript</targets>
 </desc>

aql/desc.xml

@@ -1,4 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <desc xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../_scripts/desc.xsd">
-   <targets>CSharp;Cpp;Dart;Go;Java;JavaScript;TypeScript;Python3</targets>
+   <antlr-version>^4.10</antlr-version>
+   <targets>CSharp;Cpp;Dart;Go;Java;JavaScript;Python3;TypeScript</targets>
 </desc>

argus/desc.xml

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <desc xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../_scripts/desc.xsd">
-   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;TypeScript;PHP;Python3</targets>
+   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;PHP;Python3;TypeScript</targets>
 </desc>

arithmetic/desc.xml

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <desc xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../_scripts/desc.xsd">
-   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;TypeScript;PHP;Python3</targets>
+   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;PHP;Python3;TypeScript</targets>
 </desc>

asl/desc.xml

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <desc xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../_scripts/desc.xsd">
-   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;TypeScript;PHP;Python3</targets>
+   <targets>Antlr4cs;CSharp;Cpp;Dart;Go;Java;JavaScript;PHP;Python3;TypeScript</targets>
 </desc>