CI, other bugs and improvements.

- add reusable flush solution
- sort gradle files by inclusion order
- fix CI pipeline
- disable tracing for XML driver
- make incremental gradle parts work
- restructure cp-to-solution script (and include source code in copy)

.gitlab-ci.yml

 stages:
   - build
-#  - test
-  - benchmark
+  - test
 
 before_script:
   - export GRADLE_USER_HOME=`pwd`/.gradle
@@ -10,6 +9,7 @@ cache:
   paths:
     - .gradle/wrapper
     - .gradle/caches
+    - ttc2018liveContest
 
 build:
   image: openjdk:11
@@ -17,25 +17,16 @@ build:
   script:
     - ./gradlew --console=plain assemble
 
-#test:
-#  image: openjdk:11
-#  stage: test
-#  needs:
-#    - build
-#  script:
-#    - ./gradlew --console=plain --info test
-#  artifacts:
-#    when: always
-#    reports:
-#      junit: "*/build/test-results/test/**/TEST-*.xml"
-
-timing:
+test:
   image: openjdk:11
-  stage: benchmark
+  stage: test
   needs:
     - build
   script:
-    - ./all-timed-benchmark.sh
+    - "[ -d ttc2018liveContest ] || git clone --depth=1 https://github.com/TransformationToolContest/ttc2018liveContest.git ttc2018liveContest"
+    - ln -sf ../../../../ttc2018liveContest/models solve/src/test/resources
+    - ./gradlew --console=plain test
   artifacts:
-    paths:
-      - results/
+    when: always
+    reports:
+      junit: "*/build/test-results/test/**/TEST-*.xml"

all-timed-benchmark.sh

@@ -3,6 +3,6 @@ mkdir -p "results"
 query=2
 size=32
 for d in relast-* namelookup-xml-*; do
-  ./timedBenchmark.sh $d $query $size
-  mv events-xml-${size}-Q${query}.csv results/events-${d}-${size}-Q${query}.csv
+  ./gradlew ":${d}:installDist"
+  ./timedBenchmark.sh "$d" $query $size > "results/results-${d}-${size}-Q${query}.csv"
 done

cp-to-solution.sh

-#!/usr/bin/env bash
-
-function build_sed_activate() {
-  # enable integration
-  sed -i 's/enabled = false/enabled = true/g' solve/build.gradle
-  # disable debug
-  sed -i 's|"--debug",|//"--debug",|g' solve/build.gradle
-}
-
-function build_sed_deactivate() {
-  sed -i 's/enabled = true/enabled = false/g' solve/build.gradle
-  sed -i 's|//"--debug",|"--debug",|g' solve/build.gradle
-  sed -i 's/"--incremental=[^"]*"/"--incremental=param,debug"/g' solve/build.gradle
-  sed -i 's/--flush=[^"]*/--flush=api/g' solve/build.gradle
-}
-
-function build_incremental() {
-  #TODO
-  name="$1-inc"
-  echo ">> Building $name"
-  build_sed_activate
-  sed -i 's/--flush=full/--flush=api/g' solve/build.gradle
-  sed -i 's/"--incremental=[^"]*"/"--incremental=param"/g' solve/build.gradle
-  ./gradlew --console=plain distTar
-  build_sed_deactivate
-  copy_to_ttc $name
-}
-
-function build_flushing() {
-  #TODO
-  name="$1-flush"
-  echo ">> Building $name"
-  build_sed_activate
-  sed -i 's/--flush=api/--flush=full/g' solve/build.gradle
-  sed -i 's/"--incremental=[^"]*"/"--incremental=none"/g' solve/build.gradle
-  ./gradlew --console=plain distTar
-  build_sed_deactivate
-  copy_to_ttc $name
-}
-
-function copy_to_ttc() {(
-  name=$1
-  ls -lh solve/build/distributions
-
-  # ensure directory exists and unpack into it
-  mkdir -p ../ttc2018liveContest/solutions/$name
-  tar -xf solve/build/distributions/solve-1.0-SNAPSHOT.tar -C ../ttc2018liveContest/solutions/$name/ solve-1.0-SNAPSHOT/bin/ solve-1.0-SNAPSHOT/lib/
-
-  # cp solution.ini
-  cp solution.ini ../ttc2018liveContest/solutions/$name/
-
-  # fix directory layout
-  cd ../ttc2018liveContest/solutions/$name/
-  rm -r bin/ lib/
-  mv solve-1.0-SNAPSHOT/bin/ .
-  mv solve-1.0-SNAPSHOT/lib/ .
-  rmdir solve-1.0-SNAPSHOT
-)}
-
-function build() {
-  build_flushing $@
-  build_incremental $@
-}
-
-function prepare_namelookup() {
-  # deactivate preprocess
-  sed -i 's|^generateAst.dependsOn preprocess|//generateAst.dependsOn preprocess|g' solve/build.gradle
-  # remove generated AST and helper files, activate namelookup AST and helper jadd files
-  ( cd solve/src/main/jastadd &&
-    rm SocialNetworkGen.ast SocialNetworkGen.jadd &&
-    mv Refs.jadd.unused Refs.jadd &&
-    mv SocialNetworkNamelookup.ast.unused SocialNetworkNamelookup.ast )
-}
-
-function revert_namelookup() {
-  # activate preprocess
-  sed -i 's|//generateAst.dependsOn preprocess|generateAst.dependsOn preprocess|g' solve/build.gradle
-  # deactivate normal AST and helper jadd files
-  ( cd solve/src/main/jastadd &&
-    mv Refs.jadd Refs.jadd.unused &&
-    mv SocialNetworkNamelookup.ast SocialNetworkNamelookup.ast.unused )
-}
-
-prepare_namelookup
-# Build jastadd-ttc18-xml (namelookup)
-sed -i 's/LiveContestDriverEMF/LiveContestDriverXml/' solve/build.gradle
-build "jastadd-ttc18-xml"
-
-revert_namelookup
-
-# workaround for failing build
-echo ">> Prepare building relast building"
-sleep 1
-./gradlew --console=plain preprocess jastadd build 2> /dev/null
-sleep 1
-./gradlew --console=plain preprocess jastadd build 2> /dev/null
-
-# Build jastadd-ttc18-xml
-sed -i 's/LiveContestDriverEMF/LiveContestDriverXml/' solve/build.gradle
-build "jastadd-ttc18-relast-xml"
-
-# Build jastadd-ttc18-emf
-sed -i 's/LiveContestDriverXml/LiveContestDriverEMF/' solve/build.gradle
-build "jastadd-ttc18-relast-emf"
+target="../ttc2018liveContest/solutions"
+# skipping "namelookup-xml-*"
+for s in relast-* ; do
+  echo ">> Building $s"
+  ./gradlew ":${s}:installDist"
+  solutionDir="$target/jastadd-$s"
+  echo ">> Copy to $solutionDir"
+  rm -rI "$solutionDir"
+  mkdir -p "$solutionDir/bin"
+  cp solution.ini "$solutionDir/"
+  cp "solve/README-for-TTC-repo.md" "$solutionDir/README.md"
+  cp "${s}/build/libs/solve-1.0-SNAPSHOT.jar" "${solutionDir}/bin/solve.jar"
+done
+
+# source code
+echo "Copying source"
+copySourceDir="$target/jastadd-source"
+rm -rI "$copySourceDir"
+mkdir -p "$copySourceDir/src"
+cp -a solve/gradle-parts solve/build.gradle solve/jastadd_modules solve/.gitignore "$copySourceDir/"
+cp -a solve/src/main "$copySourceDir/src/"
+cp "solve/README-for-TTC-repo.md" "$copySourceDir/README.md"

namelookup-xml-flush/build.gradle

@@ -14,8 +14,8 @@ plugins {
 
 ext.relastFiles = []
 
-apply from: '../solve/gradle-parts/base.gradle'
+apply from: '../solve/gradle-parts/01-inc-flush.gradle'
 
-apply from: '../solve/gradle-parts/inc-flush.gradle'
+apply from: '../solve/gradle-parts/02-base.gradle'
 
-apply from: '../solve/gradle-parts/rel-namelookup.gradle'
+apply from: '../solve/gradle-parts/03-rel-namelookup.gradle'

namelookup-xml-inc/build.gradle

@@ -14,8 +14,8 @@ plugins {
 
 ext.relastFiles = []
 
-apply from: '../solve/gradle-parts/base.gradle'
+apply from: '../solve/gradle-parts/01-inc-incremental.gradle'
 
-apply from: '../solve/gradle-parts/inc-incremental.gradle'
+apply from: '../solve/gradle-parts/02-base.gradle'
 
-apply from: '../solve/gradle-parts/rel-namelookup.gradle'
+apply from: '../solve/gradle-parts/03-rel-namelookup.gradle'

relast-reusable-flush/.gitignore

+../solve/.gitignore
\ No newline at end of file

relast-reusable-flush/build.gradle

+buildscript {
+    repositories.mavenCentral()
+    dependencies {
+        classpath group: 'org.jastadd', name: 'jastaddgradle', version: '1.13.3'
+    }
+}
+
+plugins {
+    id 'application'
+    id 'maven'
+    id 'java'
+    id 'idea'
+}
+
+project.ext.relastFiles = ["src/main/jastadd/base/SocialNetwork.relast",
+                   "src/main/jastadd/reusable/dg/DependencyGraph.relast",
+                   "src/main/jastadd/reusable/CommentToDG.relast"]
+
+apply from: '../solve/gradle-parts/01-inc-flush.gradle'
+
+apply from: '../solve/gradle-parts/02-base.gradle'
+
+apply from: '../solve/gradle-parts/03-rel-relast.gradle'

relast-reusable-flush/jastadd_modules

+module("solve") {
+    jastadd {
+        basedir "src/"
+        include "main/jastadd/base/**/*.ast"
+        include "main/jastadd/base/**/*.jadd"
+        include "main/jastadd/base/**/*.jrag"
+        include "main/jastadd/reusable/dg/DependencyGraphReachability.jrag"
+        include "main/jastadd/reusable/CommentToDG.jrag"
+        include "main/jastadd/reusable/queriesRefined.jrag"
+        include "gen/jastadd/**/*.ast"
+        include "gen/jastadd/**/*.jadd"
+    }
+}

relast-reusable-flush/libs

+../solve/libs
\ No newline at end of file

relast-reusable-flush/src/main

+../../solve/src/main
\ No newline at end of file

relast-reusable-flush/src/test

+../../solve/src/test
\ No newline at end of file

relast-reusable-inc/build.gradle

@@ -16,8 +16,8 @@ project.ext.relastFiles = ["src/main/jastadd/base/SocialNetwork.relast",
                    "src/main/jastadd/reusable/dg/DependencyGraph.relast",
                    "src/main/jastadd/reusable/CommentToDG.relast"]
 
-apply from: '../solve/gradle-parts/base.gradle'
+apply from: '../solve/gradle-parts/01-inc-incremental.gradle'
 
-apply from: '../solve/gradle-parts/inc-incremental.gradle'
+apply from: '../solve/gradle-parts/02-base.gradle'
 
-apply from: '../solve/gradle-parts/rel-relast.gradle'
+apply from: '../solve/gradle-parts/03-rel-relast.gradle'

relast-xml-flush/build.gradle

@@ -14,8 +14,8 @@ plugins {
 
 ext.relastFiles = ["src/main/jastadd/base/SocialNetwork.relast"]
 
-apply from: '../solve/gradle-parts/base.gradle'
+apply from: '../solve/gradle-parts/01-inc-flush.gradle'
 
-apply from: '../solve/gradle-parts/inc-flush.gradle'
+apply from: '../solve/gradle-parts/02-base.gradle'
 
-apply from: '../solve/gradle-parts/rel-relast.gradle'
+apply from: '../solve/gradle-parts/03-rel-relast.gradle'

relast-xml-inc/build.gradle

@@ -14,8 +14,8 @@ plugins {
 
 ext.relastFiles = ["src/main/jastadd/base/SocialNetwork.relast"]
 
-apply from: '../solve/gradle-parts/base.gradle'
+apply from: '../solve/gradle-parts/01-inc-incremental.gradle'
 
-apply from: '../solve/gradle-parts/inc-incremental.gradle'
+apply from: '../solve/gradle-parts/02-base.gradle'
 
-apply from: '../solve/gradle-parts/rel-relast.gradle'
+apply from: '../solve/gradle-parts/03-rel-relast.gradle'

settings.gradle

 include ':solve'
 include ':namelookup-xml-flush'
 include ':namelookup-xml-inc'
+include ':relast-reusable-flush'
 include ':relast-reusable-inc'
 include ':relast-xml-flush'
 include ':relast-xml-inc'

solution.ini

@@ -3,5 +3,5 @@ default=echo "It is already built"
 skipTests=echo "It is already built"
 
 [run]
-Q1=bin/solve
-Q2=bin/solve
+Q1=java -Xms6G -Xmx6G -jar bin/solve.jar
+Q2=java -Xms6G -Xmx6G -jar bin/solve.jar

solve/README-for-TTC-repo.md

+The sources for all JastAdd solutions can be found in a [separate repository](https://git-st.inf.tu-dresden.de/stgroup/ttc18live).

solve/build.gradle

@@ -14,8 +14,8 @@ plugins {
 
 ext.relastFiles = ["src/main/jastadd/base/SocialNetwork.relast"]
 
-apply from: '../solve/gradle-parts/base.gradle'
+apply from: 'gradle-parts/01-inc-incremental.gradle'
 
-apply from: '../solve/gradle-parts/inc-incremental.gradle'
+apply from: 'gradle-parts/02-base.gradle'
 
-apply from: '../solve/gradle-parts/rel-relast.gradle'
+apply from: 'gradle-parts/03-rel-relast.gradle'

solve/gradle-parts/01-inc-flush.gradle

+project.ext.extraJastAddOptions = [
+        '--lineColumnNumbers',
+        '--cache=all',
+        "--flush=full",
+        "--incremental=none",
+        "--tracing=api",
+]