Resolve "Change JastAdd version to updated, packaged one"

- use packaged jastadd2 (dresden version), relast, preprocessor
- remove relast-preprocessor
- update preprocessor.testing to work with packaged relast
- add base as additional dependency for tests because of Errors test (as there, the compiler is invoked directly)
- update log4j
- use standardized build.gradle structure
- build non-master version with appended CI pipeline IID

.gitlab-ci.yml

@@ -46,7 +46,18 @@ test:
       junit: "ragconnect.tests/build/test-results/**/TEST-*.xml"
     expire_in: 1 week
 
-publish:
+publish-dev:
+  image: openjdk:11
+  stage: publish
+  needs:
+    - test
+  script:
+    - "./gradlew setDevVersionForCI"
+    - "./gradlew publish"
+  except:
+    - master
+
+publish-master:
   image: openjdk:11
   stage: publish
   needs:
@@ -54,7 +65,6 @@ publish:
   script:
     - "./gradlew publish"
   only:
-    - dev
     - master
 
 ragdoc_build:

.gitmodules

-[submodule "relast-preprocessor"]
-	path = relast-preprocessor
-	url = ../relast-preprocessor.git
-	branch = develop

build.gradle

+plugins {
+    id 'com.github.ben-manes.versions' version '0.42.0'
+    id 'java'
+}
+
+java.toolchain.languageVersion.set(JavaLanguageVersion.of(11))

gradle.properties

-log4j_version = 2.14.0
-mustache_java_version = 0.9.7
+log4j_version = 2.17.2
+mustache_java_version = 0.9.10
+preprocessor_version = 0.1.0-41

ragconnect.base/build.gradle

+// --- Buildscripts (must be at the top) ---
 buildscript {
     repositories.mavenCentral()
     dependencies {
@@ -5,21 +6,18 @@ buildscript {
     }
 }
 
+// --- Plugin definitions ---
 plugins {
+    id 'com.github.ben-manes.versions'
     id 'java'
-    id 'java-library'
     id 'idea'
-    id 'com.github.ben-manes.versions' version '0.36.0'
-    id 'maven-publish'
+    id 'org.jastadd'
     id 'application'
+    id 'java-library'
+    id 'maven-publish'
 }
 
-apply plugin: 'jastadd'
-
-group = 'de.tudresden.inf.st'
-
-mainClassName = 'org.jastadd.ragconnect.compiler.Compiler'
-
+// --- Dependencies ---
 repositories {
     mavenCentral()
     maven {
@@ -27,76 +25,61 @@ repositories {
         url "https://git-st.inf.tu-dresden.de/api/v4/groups/jastadd/-/packages/maven"
     }
 }
-tasks.compileJava {
-    options.release.set(11)
+
+configurations {
+    relast
 }
 
 dependencies {
-    implementation project(':relast-preprocessor')
+    relast group: 'org.jastadd', name: 'relast', version: "0.3.0-137"
+    implementation group: 'org.jastadd', name: 'relast-preprocessor', version: "${preprocessor_version}"
     implementation group: 'com.github.spullara.mustache.java', name: 'compiler', version: "${mustache_java_version}"
     runtimeOnly group: 'org.jastadd', name: 'jastadd2', version: '2.3.5-dresden'
-//    runtimeOnly fileTree(include: ['jastadd2.jar'], dir: '../libs')
     api group: 'net.sf.beaver', name: 'beaver-rt', version: '0.9.11'
 }
 
-java {
-    toolchain {
-        languageVersion.set(JavaLanguageVersion.of(11))
-    }
-}
-
-def versionFile = 'src/main/resources/ragConnectVersion.properties'
-def props = new Properties()
-
-try {
-    file(versionFile).withInputStream { stream -> props.load(stream) }
-    version = props['version']
-} catch (e) {
-    // this happens, if either the properties file is not present, or cannot be read from
-    throw new GradleException("File ${versionFile} not found or unreadable. Aborting.", e)
-}
-
-task printVersion() {
-    doLast {
-        println(version)
-    }
-}
-
-task newVersion() {
-    doFirst {
-        def newProps = new Properties()
-        newProps['version'] = value
-        newProps.store(file(versionFile).newWriter(), null)
-    }
+// --- Preprocessors ---
+ext {
+    extractLocation = "src/gen/jastadd-sources/relast.preprocessor"
 }
-
 File genSrc = file("src/gen/java")
 sourceSets.main.java.srcDir genSrc
 idea.module.generatedSourceDirs += genSrc
 
-jar {
-    manifest {
-        attributes "Main-Class": 'org.jastadd.ragconnect.compiler.Compiler'
+task extractJastAddSources(type: Sync) {
+    dependsOn configurations.runtimeClasspath
+    configurations.runtimeClasspath.asFileTree.filter { it.toString().endsWith("relast-preprocessor-${preprocessor_version}.jar") }.collect {
+        from(zipTree(it)) {
+            include "**/*.jrag"
+            include "**/*.jadd"
+            include "**/*.ast"
+            include "**/*.relast"
+            include "**/*.flex"
+            include "**/*.parser"
         }
-
-    from {
-        configurations.runtimeClasspath.collect { it.isDirectory() ? it : zipTree(it) }
     }
-    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
-
-    archiveBaseName = 'ragconnect'
+    includeEmptyDirs false
+    into file("${extractLocation}")
 }
 
-File preprocessorGrammar = file('../relast-preprocessor/src/main/jastadd/RelAst.relast')
+File preprocessorGrammar = file("${extractLocation}/RelAst.relast")
 File ragConnectGrammar = file('./src/main/jastadd/RagConnect.relast')
 File intermediateGrammar = file('./src/main/jastadd/Intermediate.relast')
-File mustacheGrammar = file('../relast-preprocessor/src/main/jastadd/mustache/Mustache.relast')
+File mustacheGrammar = file("${extractLocation}/mustache/Mustache.relast")
+
 task relast(type: JavaExec) {
     group = 'Build'
-    main = "-jar"
+    classpath = configurations.relast
+    //noinspection GroovyAssignabilityCheck, GroovyAccessibility
+    mainClass = 'org.jastadd.relast.compiler.Compiler'
+
+    dependsOn extractJastAddSources
+
+    doFirst {
+        mkdir "src/gen/jastadd/"
+    }
 
     args = [
-            "../libs/relast.jar",
             preprocessorGrammar,
             ragConnectGrammar,
             intermediateGrammar,
@@ -128,52 +111,45 @@ clean {
     delete "src/gen/jastadd/RagConnectResolverStubs.jrag"
 }
 
+// --- JastAdd ---
 jastadd {
     configureModuleBuild()
     modules {
         //noinspection GroovyAssignabilityCheck
         module("RagConnect") {
 
-            java {
-                basedir ".."
-                include "relast-preprocessor/main/**/*.java"
-                include "relast-preprocessor/gen/**/*.java"
-                include "ragconnect.base/src/main/**/*.java"
-                include "ragconnect.base/src/gen/**/*.java"
-            }
-
             jastadd {
-                basedir ".."
-                include "relast-preprocessor/src/main/jastadd/**/*.ast"
-                include "relast-preprocessor/src/main/jastadd/**/*.jadd"
-                include "relast-preprocessor/src/main/jastadd/**/*.jrag"
-                include "ragconnect.base/src/main/jastadd/**/*.ast"
-                include "ragconnect.base/src/main/jastadd/**/*.jadd"
-                include "ragconnect.base/src/main/jastadd/**/*.jrag"
-                include "ragconnect.base/src/gen/jastadd/**/*.ast"
-                include "ragconnect.base/src/gen/jastadd/**/*.jadd"
-                include "ragconnect.base/src/gen/jastadd/**/*.jrag"
+                basedir "."
+                include "src/gen/jastadd-sources/relast.preprocessor/**/*.ast"
+                include "src/gen/jastadd-sources/relast.preprocessor/**/*.jadd"
+                include "src/gen/jastadd-sources/relast.preprocessor/**/*.jrag"
+                include "src/main/jastadd/**/*.ast"
+                include "src/main/jastadd/**/*.jadd"
+                include "src/main/jastadd/**/*.jrag"
+                include "src/gen/jastadd/**/*.ast"
+                include "src/gen/jastadd/**/*.jadd"
+                include "src/gen/jastadd/**/*.jrag"
             }
 
             scanner {
-                basedir ".."
-                include "ragconnect.base/src/main/jastadd/scanner/Header.flex",             [-5]
-                include "relast-preprocessor/src/main/jastadd/scanner/Preamble.flex",       [-4]
-                include "relast-preprocessor/src/main/jastadd/scanner/Macros.flex",         [-3]
-                include "ragconnect.base/src/main/jastadd/scanner/Macros.flex",             [-3]
-                include "relast-preprocessor/src/main/jastadd/scanner/RulesPreamble.flex",  [-2]
-                include "ragconnect.base/src/main/jastadd/scanner/MappingContent.flex",     [-1]
-                include "ragconnect.base/src/main/jastadd/scanner/Keywords.flex"
-                include "relast-preprocessor/src/main/jastadd/scanner/Keywords.flex"
-                include "relast-preprocessor/src/main/jastadd/scanner/Symbols.flex",        [1]
-                include "relast-preprocessor/src/main/jastadd/scanner/RulesPostamble.flex", [2]
+                basedir "."
+                include "src/main/jastadd/scanner/Header.flex",                                    [-5]
+                include "src/gen/jastadd-sources/relast.preprocessor/scanner/Preamble.flex",       [-4]
+                include "src/gen/jastadd-sources/relast.preprocessor/scanner/Macros.flex",         [-3]
+                include "src/main/jastadd/scanner/Macros.flex",                                    [-3]
+                include "src/gen/jastadd-sources/relast.preprocessor/scanner/RulesPreamble.flex",  [-2]
+                include "src/main/jastadd/scanner/MappingContent.flex",                            [-1]
+                include "src/main/jastadd/scanner/Keywords.flex"
+                include "src/gen/jastadd-sources/relast.preprocessor/scanner/Keywords.flex"
+                include "src/gen/jastadd-sources/relast.preprocessor/scanner/Symbols.flex",        [ 1]
+                include "src/gen/jastadd-sources/relast.preprocessor/scanner/RulesPostamble.flex", [ 2]
             }
 
             parser {
-                basedir ".."
-                include "ragconnect.base/src/main/jastadd/parser/Preamble.parser"
-                include "ragconnect.base/src/main/jastadd/parser/RagConnect.parser"
-                include "relast-preprocessor/src/main/jastadd/parser/RelAst.parser"
+                basedir "."
+                include "src/main/jastadd/parser/Preamble.parser"
+                include "src/main/jastadd/parser/RagConnect.parser"
+                include "src/gen/jastadd-sources/relast.preprocessor/parser/RelAst.parser"
             }
         }
     }
@@ -184,9 +160,7 @@ jastadd {
     }
 
     preprocessParser.doFirst {
-
         args += ["--no-beaver-symbol"]
-
     }
 
     module = "RagConnect"
@@ -205,33 +179,83 @@ jastadd {
     jastaddOptions = ["--lineColumnNumbers", "--List=JastAddList", "--safeLazy", "--visitCheck=true", "--rewrite=cnta", "--cache=all"]
 }
 
-generateAst.dependsOn relast
+// --- Versioning and Publishing ---
+group = 'de.tudresden.inf.st'
+
+ext {
+    mainClassName = 'org.jastadd.ragconnect.compiler.Compiler'
+}
+application.mainClassName = "${mainClassName}"
+
+jar {
+    manifest.attributes "Main-Class": "${mainClassName}"
+}
+
+task fatJar(type: Jar) {
+    dependsOn jar
+    group = "build"
+    //noinspection GroovyAssignabilityCheck, GroovyAccessibility
+    archiveAppendix = "fatjar"
+    from sourceSets.main.output
+    from {
+        configurations.runtimeClasspath.collect {it.isDirectory() ? it : zipTree(it) }
+    }
+
+    manifest.attributes "Main-Class": "${mainClassName}"
+}
+
+def versionFile = "src/main/resources/ragconnectVersion.properties"
+
+try {
+    def props = new Properties()
+    file(versionFile).withInputStream { stream -> props.load(stream) }
+    version = props['version']
+} catch (e) {
+    // this happens, if either the properties file is not present, or cannot be read from
+    throw new GradleException("File ${versionFile} not found or unreadable. Aborting.", e)
+}
+
+task printVersion() {
+    doLast {
+        println(version)
+    }
+}
+
+task newVersion() {
+    doFirst {
+        def newProps = new Properties()
+        newProps['version'] = value
+        newProps.store(file(versionFile).newWriter(), null)
+    }
+}
+
+task setDevVersionForCI() {
+    doFirst {
+        def props = new Properties()
+        props['version'] = version + "-$System.env.CI_PIPELINE_IID"
+        props.store(file(versionFile).newWriter(), null)
+    }
+}
 
-//708
 publishing {
     publications {
+        //noinspection GroovyAssignabilityCheck
         maven(MavenPublication) {
+            //noinspection GroovyAssignabilityCheck
             artifactId = 'ragconnect'
-            // Comment rs: components.java does not include relast.preprocessor
-            // from components.java
-            artifact("build/libs/ragconnect-${version}.jar") {
-                extension 'jar'
-            }
+            //noinspection GroovyAssignabilityCheck
+            from components.java
         }
     }
     repositories {
         maven {
-            url "https://git-st.inf.tu-dresden.de/api/v4/projects/708/packages/maven"
-            // Uncomment the following lines to publish manually (and comment out the other credentials section)
-//            credentials(HttpHeaderCredentials) {
-//                name = "Private-Token"
-//                value = gitLabPrivateToken // the variable resides in ~/.gradle/gradle.properties
-//            }
+            url "https://git-st.inf.tu-dresden.de/api/v4/projects/$System.env.CI_PROJECT_ID/packages/maven"
             credentials(HttpHeaderCredentials) {
                 name = 'Job-Token'
                 value = System.getenv("CI_JOB_TOKEN")
             }
             authentication {
+                //noinspection GroovyAssignabilityCheck
                 header(HttpHeaderAuthentication)
             }
         }
@@ -239,5 +263,6 @@ publishing {
     }
 }
 
+// --- Task order ---
+generateAst.dependsOn relast
 publish.dependsOn jar
-jar.dependsOn ":relast-preprocessor:jar"

ragconnect.base/src/main/java/org/jastadd/ragconnect/compiler/Compiler.java

@@ -154,7 +154,7 @@ public class Compiler extends AbstractCompiler {
    */
   private String readVersion() {
     try {
-      ResourceBundle resources = ResourceBundle.getBundle("ragConnectVersion");
+      ResourceBundle resources = ResourceBundle.getBundle("ragconnectVersion");
       return resources.getString("version");
     } catch (MissingResourceException e) {
       return "version ?";

ragconnect.tests/build.gradle

+// --- Buildscripts (must be at the top) ---
 buildscript {
     repositories {
         mavenCentral()
-//        mavenLocal()
         maven {
             name "gitlab-maven"
             url "https://git-st.inf.tu-dresden.de/api/v4/groups/jastadd/-/packages/maven"
@@ -9,26 +9,26 @@ buildscript {
     }
     dependencies {
         classpath 'org.jastadd:jastaddgradle:1.13.3'
-        classpath 'org.jastadd.preprocessor:testing:0.2.12'
+        classpath 'org.jastadd.preprocessor:testing:0.3.0-37'
     }
 }
 
+// --- Plugin definitions ---
 import org.jastadd.preprocessor.testing.plugin.PreprocessorPlugin
 import org.jastadd.preprocessor.testing.plugin.RagConnectTest
 
 plugins {
+    id 'com.github.ben-manes.versions'
     id 'java'
-    id 'java-library'
     id 'idea'
-    id 'com.github.ben-manes.versions' version '0.36.0'
+    id 'org.jastadd'
+    id 'java-library'
     id 'com.google.protobuf' version "0.8.14"
 }
 
-apply plugin: 'jastadd'
 apply plugin: PreprocessorPlugin
 
-group = 'de.tudresden.inf.st'
-
+// --- Dependencies ---
 repositories {
     mavenCentral()
     maven {
@@ -36,15 +36,18 @@ repositories {
         url "https://git-st.inf.tu-dresden.de/api/v4/groups/jastadd/-/packages/maven"
     }
 }
-tasks.compileTestJava {
-    options.release.set(11)
+
+configurations {
+    ragconnect
+    relast
 }
 
 dependencies {
-    implementation project(':ragconnect.base')
+    ragconnect project(':ragconnect.base')
+    testImplementation project(':ragconnect.base')
 
-//    runtimeOnly group: 'org.jastadd', name: 'jastadd', version: '2.3.5-dresden'
-    runtimeOnly fileTree(include: ['jastadd2.jar'], dir: '../libs')
+    implementation group: 'org.jastadd', name: 'jastadd2', version: '2.3.5-dresden-5'
+    relast group: 'org.jastadd', name: 'relast', version: "0.3.0-137"
 
     testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.4.0'
     testRuntimeOnly group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.4.0'
@@ -69,19 +72,17 @@ dependencies {
     api group: 'com.google.protobuf', name: 'protobuf-java', version: '3.0.0'
 }
 
-java {
-    toolchain {
-        languageVersion.set(JavaLanguageVersion.of(11))
-    }
-}
+// --- Preprocessors ---
+File genSrc = file("src/test/java-gen")
+sourceSets.test.java.srcDir genSrc
+idea.module.generatedSourceDirs += genSrc
 
-//task helper {
-//    doLast {
-//        println(defaultOnlyRead.inputs.files.files)
-//        println(defaultOnlyRead.outputs.files.files)
-//    }
-//}
+preprocessorTesting {
+    relastCompilerConfiguration = configurations.relast
+    ragconnectCompilerConfiguration = configurations.ragconnect
+}
 
+// --- Tests ---
 test {
     useJUnitPlatform {
         excludeTags 'mqtt'
@@ -90,13 +91,6 @@ test {
     maxHeapSize = '1G'
 }
 
-protobuf {
-    protoc {
-        // The artifact spec for the Protobuf Compiler
-        artifact = 'com.google.protobuf:protoc:3.0.0'
-    }
-}
-
 task allTests(type: Test, dependsOn: testClasses) {
     description = 'Run every test'
     group = 'verification'
@@ -126,16 +120,6 @@ task newTests(type: Test, dependsOn: testClasses) {
     }
 }
 
-preprocessorTesting {
-    //noinspection GroovyAssignabilityCheck
-    relastCompilerLocation = '../libs/relast.jar'
-//    ragconnectCompilerLocation = '../libs/ragconnect.jar'
-}
-
-File genSrc = file("src/test/java-gen")
-sourceSets.test.java.srcDir genSrc
-idea.module.generatedSourceDirs += genSrc
-
 clean {
     delete fileTree(dir: 'src/test/02-after-ragconnect/', exclude: '.gitignore')
     delete fileTree(dir: 'src/test/03-after-relast/', exclude: '.gitignore')
@@ -151,8 +135,6 @@ def JASTADD_INCREMENTAL_OPTIONS_TRACING_FULL = JASTADD_INCREMENTAL_OPTIONS.clone
 JASTADD_INCREMENTAL_OPTIONS_TRACING_FULL.set(0, '--tracing=all')
 JASTADD_INCREMENTAL_OPTIONS_TRACING_FULL.set(1, '--incremental=param,debug')
 
-classes.dependsOn(':ragconnect.base:jar')
-
 // --- Test: Example ---
 task compileExampleTest(type: RagConnectTest) {
     ragconnect {
@@ -355,7 +337,7 @@ task compileMapping(type: RagConnectTest) {
 }
 
 // --- Test: tree-manual ---
-task compileTreeManual(type: RagConnectTest, dependsOn: ':ragconnect.base:compileJava') {
+task compileTreeManual(type: RagConnectTest) {
     ragconnect {
         outputDir = file('src/test/02-after-ragconnect/tree')
         inputFiles = [file('src/test/01-input/tree/Test.relast'),
@@ -668,6 +650,7 @@ task compileRelationIncremental(type: RagConnectTest) {
                       file('src/test/01-input/relation/Test.connect')]
         rootNode = 'Root'
         logWrites = true
+        logIncremental = true
         extraOptions = defaultRagConnectOptionsAnd(['--experimental-jastadd-329'])
     }
     relast {
@@ -682,11 +665,22 @@ task compileRelationIncremental(type: RagConnectTest) {
         extraOptions = JASTADD_INCREMENTAL_OPTIONS_TRACING_FULL
     }
 }
-compileRelationIncremental.outputs.upToDateWhen { false }
 
+// --- Task order ---
+classes.dependsOn(':ragconnect.base:jar')
+//compileAttributeIncremental.outputs.upToDateWhen { false }
+
+// --- Misc ---
 static ArrayList<String> defaultRagConnectOptionsAnd(ArrayList<String> options = []) {
     if (!options.contains('--logTarget=slf4j')) {
         options.add('--logTarget=slf4j')
     }
     return options
 }
+
+protobuf {
+    protoc {
+        // The artifact spec for the Protobuf Compiler
+        artifact = 'com.google.protobuf:protoc:3.0.0'
+    }
+}

relast-preprocessor @ cee30f5c

-Subproject commit cee30f5c8a1b356118f5ed78302a1df64fe76897

settings.gradle

@@ -6,6 +6,5 @@ pluginManagement {
 
 rootProject.name = 'ragconnect'
 
-include 'relast-preprocessor'
 include 'ragconnect.base'
 include 'ragconnect.tests'