Merge branch 'feature/pages' into 'develop'

Feature/pages

See merge request jastadd/relast-preprocessor!3

.gitlab-ci.yml

@@ -4,12 +4,27 @@ variables:
 stages:
   - build
   - test
+  - ragdoc_build
+  - ragdoc_view
+  - publish
+
+before_script:
+  - export GRADLE_USER_HOME=`pwd`/.gradle
+
+cache:
+  paths:
+    - .gradle/wrapper
+    - .gradle/caches
 
 build:
   image: openjdk:8
   stage: build
   script:
     - ./gradlew --console=plain --no-daemon assemble
+  artifacts:
+    paths:
+      - "src/gen"
+    expire_in: 1 week
 
 test:
   image: openjdk:8
@@ -19,3 +34,51 @@ test:
   artifacts:
     reports:
       junit: build/test-results/test/**/TEST-*.xml
+
+ragdoc_build:
+  image:
+    name: "git-st.inf.tu-dresden.de:4567/jastadd/ragdoc-builder"
+    entrypoint: [""]
+  stage: ragdoc_build
+  needs:
+    - build
+  script:
+    - JAVA_FILES=$(find src/ -name '*.java')
+    - /ragdoc-builder/start-builder.sh -excludeGenerated -d data/ $JAVA_FILES
+  artifacts:
+    paths:
+      - "data/"
+
+ragdoc_view:
+  image:
+    name: "git-st.inf.tu-dresden.de:4567/jastadd/ragdoc-view:relations"
+    entrypoint: [""]
+  stage: ragdoc_view
+  needs:
+    - ragdoc_build
+  script:
+    - DATA_DIR=$(pwd -P)/data
+    - mkdir -p pages/docs/ragdoc
+    - OUTPUT_DIR=$(pwd -P)/pages/docs/ragdoc
+    - cd /ragdoc-view/src/ && rm -rf data && ln -s $DATA_DIR
+    - /ragdoc-view/build-view.sh --output-path=$OUTPUT_DIR
+  artifacts:
+    paths:
+      - "pages/docs/ragdoc"
+
+pages:
+  image: python:3.8-buster
+  stage: publish
+  needs:
+    - ragdoc_view
+    - test
+  before_script:
+    - pip install -U mkdocs mkdocs-macros-plugin mkdocs-git-revision-date-localized-plugin
+  script:
+    - cd pages && mkdocs build
+#  only:
+#    - develop
+#    - master
+  artifacts:
+    paths:
+      - public

build.gradle

@@ -5,7 +5,6 @@ plugins {
     id 'java'
     id 'idea'
     id 'java-test-fixtures'
-    id 'com.github.ben-manes.versions' version '0.34.0'
 }
 
 ext {

pages/.gitignore

+/docs/ragdoc/

pages/custom_theme/footer.html

+{% block footer %}
+<p>{% if config.copyright %}
+<small>{{ config.copyright }}<br></small>
+{% endif %}
+<hr>
+Built with <a href="https://www.mkdocs.org/">MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+{% if page and page.meta and page.meta.git_revision_date_localized %}
+<small><br><i>Last updated {{ page.meta.git_revision_date_localized }}</i></small>
+{% endif %}
+</p>
+{% endblock %}
+

pages/docs/index.md

+# Relational RAGs Preprocessor Documentation
+
+The [Relational RAGs Preprocessor](https://git-st.inf.tu-dresden.de/jastadd/relast-preprocessor) is a framework to create custom preprocessors for [JastAdd](http://jastadd.org) and JastAdd-based [Relational RAGs](http://www.relational-rags.eu).
+

pages/docs/using.md

+# Creating a Preprocessor
+
+## Project Structure
+
+The *gradle* build system is required to use the preprocessor.
+It is recommended to use the same version of gradle as the preprocessor does internally, which currently is 6.7.
+
+To create a new preprocessor, follow the following steps:
+
+1. Create a new gradle project
+2. Include this repository as a submodule in the directory `relast.preprocessor`
+   - copy the contents of this repository into this directory; this can be done using a git submodule
+   - in your `settings.gradle` add a dependency to it:
+     ```groovy
+     include 'relast.preprocessor'
+     ```
+3. Adapt the `build.gradle` file of your project to use the subproject; this step is described in the next section
+
+## Constructing a Gradle Build File
+
+The preprocessor and its test infrastructure are included as project dependencies.
+
+```groovy
+dependencies {
+    implementation project(':relast.preprocessor')
+    testImplementation testFixtures(project(":relast.preprocessor"))
+}
+```
+
+Next, the RelAST preprocessor must be called using files provided by the framework and potentially other extensions of the RelAST gramamr DSL.
+
+```groovy
+// Input and output files for relast
+def relastInputFiles = [
+        "relast.preprocessor/src/main/jastadd/RelAst.relast",
+        "relast.preprocessor/src/main/jastadd/mustache/Mustache.relast"
+        // add more files here
+]
+def relastOutputFiles = [
+        "src/gen/jastadd/RelAst.ast",
+        "src/gen/jastadd/RelAst.jadd",
+        "src/gen/jastadd/RelAstRefResolver.jadd",
+        "src/gen/jastadd/RelAstResolverStubs.jrag"
+]
+
+task relast(type: JavaExec) {
+    classpath = files("relast.preprocessor/libs/relast.jar")
+    group = 'Build'
+
+    doFirst {
+        delete relastOutputFiles
+        mkdir "src/gen/jastadd/"
+    }
+
+    args = [
+            "--listClass=java.util.ArrayList",
+            "--jastAddList=JastAddList",
+            "--useJastAddNames",
+            "--file",
+            "--resolverHelper",
+            "--grammarName=./src/gen/jastadd/RelAst"
+    ] + relastInputFiles
+
+    inputs.files relastInputFiles
+    outputs.files relastOutputFiles
+}
+```
+
+Finally, JastAdd must be configured to use the correct files.
+Note that the scanner includes are split into several files to allow for an extensible lexical analysis.
+
+```groovy
+jastadd {
+    configureModuleBuild()
+    modules {
+        //noinspection GroovyAssignabilityCheck
+        module("Preprocessor") {
+
+            jastadd {
+                basedir "."
+                include "relast.preprocessor/src/main/jastadd/**/*.ast"
+                include "relast.preprocessor/src/main/jastadd/**/*.jadd"
+                include "relast.preprocessor/src/main/jastadd/**/*.jrag"
+                include "src/main/jastadd/**/*.ast"
+                include "src/main/jastadd/**/*.jadd"
+                include "src/main/jastadd/**/*.jrag"
+                include "src/gen/jastadd/**/*.ast"
+                include "src/gen/jastadd/**/*.jadd"
+                include "src/gen/jastadd/**/*.jrag"
+            }
+
+            scanner {
+                include "relast.preprocessor/src/main/jastadd/scanner/Header.flex",         [-4]
+                include "relast.preprocessor/src/main/jastadd/scanner/Preamble.flex",       [-3]
+                include "relast.preprocessor/src/main/jastadd/scanner/Macros.flex",         [-2]
+                include "relast.preprocessor/src/main/jastadd/scanner/RulesPreamble.flex",  [-1]
+                include "relast.preprocessor/src/main/jastadd/scanner/Keywords.flex",       [ 0]
+                include "relast.preprocessor/src/main/jastadd/scanner/Symbols.flex",        [ 1]
+                include "relast.preprocessor/src/main/jastadd/scanner/RulesPostamble.flex", [ 2]
+            }
+
+            parser {
+                include "relast.preprocessor/src/main/jastadd/parser/Preamble.parser"
+                include "relast.preprocessor/src/main/jastadd/parser/RelAst.parser"
+            }
+        }
+    }
+
+    cleanGen.doFirst {
+        delete "src/gen"
+        delete "src/gen-res"
+    }
+
+    preprocessParser.doFirst {
+        args += ["--no-beaver-symbol"]
+    }
+
+    module = "Preprocessor"
+    astPackage = 'org.jastadd.relast.ast'
+    parser.name = 'RelAstParser'
+    genDir = 'src/gen/java'
+    buildInfoDir = 'src/gen-res'
+    scanner.genDir = "src/gen/java/org/jastadd/relast/scanner"
+    parser.genDir = "src/gen/java/org/jastadd/relast/parser"
+    jastaddOptions = ["--lineColumnNumbers", "--List=JastAddList", "--safeLazy", "--visitCheck=true", "--rewrite=cnta", "--cache=all"]
+}
+```
+
+## Writing Tests
+
+The preprocessor provides a test framework based on on JUnit 5.
+This framework can be used to write acceptance tests, comparing input and output of runs of the constructed preprocessor.
+Hereby, no java code has to be written - only a simple configuration file and some input and expected data have to be provided. 
+
+### Test Class
+
+Create a new class inheriting from `RelAstProcessorTestBase` and set the `mainClass` field to the correct class.
+
+```java
+import org.jastadd.relast.tests.RelAstProcessorTestBase;
+import org.junit.jupiter.api.BeforeAll;
+
+public class MyPreprocessorTest extends RelAstProcessorTestBase {
+  @BeforeAll
+  static void init() {
+    mainClass = Main.class;
+  }
+}
+```
+
+### Test DSL
+
+Tests are described by a config file. Each directory in `src/main/resources` containing a `config.yaml` is assumed to contain test data.
+
+A test configuration must contain a list of objects containing the following properties:
+
+| Name           | Type                  | Required | Default Value | Description                                          |
+| -------------- | --------------------- | -------- | ------------- | ---------------------------------------------------- |
+|                |                       |          |               |                                                      |
+| `name`         | string                | yes      | --            | name of the test case                                |
+| `args`         | list of strings       | yes      | --            | arguments passed to the preprocessor                 |
+| `fail`         | boolean               | no       | `false`       | is the test a negative test                          |
+|                |                       |          |               |                                                      |
+| `in`           | string                | no       | `"in"`        | directory containing input data                      |
+| `out`          | string                | no       | `"out"`       | directory the output is written to                   |
+| `expected`     | string                | no       | `"expected"`  | directory containing expected data                   |
+|                |                       |          |               |                                                      |
+| `compare`      | boolean               | no       | `false`       | should `out` and `expected` directories be compared? |
+|                |                       |          |               |                                                      |
+| `out-contains` | list of strings       | no       | empty list    | string that must be contained in the `stdout` stream |
+| `err-contains` | list of strings       | no       | empty list    | string that must be contained in the `stderr` stream |
+| `out-matches`  | list of regex strings | no       | empty list    | regular expression the `stdout` stream must match    |
+| `err-matches`  | list of regex strings | no       | empty list    | regular expression the `stderr` stream must match    |
+|                |                       |          |               |                                                      |
+
+Example:
+
+```yaml
+- name: "SimpleInheritance (null)"
+  compare: true
+  out: "null/out"
+  expected: "null/expected"
+  args:
+    - "--inputBaseDir=in"
+    - "--outputBaseDir=null/out"
+    - "--printYaml"
+    - "--errorHandling=null"
+    - "Example.relast"
+- name: "SimpleInheritance (wrong exception)"
+  fail: true
+  err-matches:
+    - "(?s).*Invalid.*"
+  err-contains:
+    - "Invalid argument 'not.a.Class' for parameter 'errorHandling': Class not found (name must be qualified)."
+  args:
+    - "--inputBaseDir=in"
+    - "--outputBaseDir=out/does_not_matter_will_fail"
+    - "--errorHandling=not.a.Class"
+    - "Example.relast"
+```

pages/mkdocs.yml

+site_name: Relational RAGs Preprocessor Documentation
+nav:
+  - using.md
+  - API documentation: ragdoc/index.html
+theme:
+  name: readthedocs
+  custom_dir: custom_theme/
+plugins:
+  - search
+  - git-revision-date-localized:
+      type: datetime
+      timezone: Europe/Berlin
+      locale: en
+      fallback_to_build_date: True
+  - macros
+repo_url: https://git-st.inf.tu-dresden.de/jastadd/relast-preprocessor
+site_dir: ../public
+

src/main/resources/preprocessor.properties

-version=1.0.0-develop
+version=1.0.0-pre-release