From 3a847a6612ed71252c0f11dc245328b730695a55 Mon Sep 17 00:00:00 2001 From: Johannes Mey <johannes.mey@tu-dresden.de> Date: Sun, 7 Feb 2021 12:18:12 +0000 Subject: [PATCH] Feature/pages --- .gitlab-ci.yml | 63 +++++++ build.gradle | 1 - pages/.gitignore | 1 + pages/custom_theme/footer.html | 12 ++ pages/docs/index.md | 4 + pages/docs/using.md | 200 +++++++++++++++++++++ pages/mkdocs.yml | 18 ++ src/main/resources/preprocessor.properties | 2 +- 8 files changed, 299 insertions(+), 2 deletions(-) create mode 100644 pages/.gitignore create mode 100644 pages/custom_theme/footer.html create mode 100644 pages/docs/index.md create mode 100644 pages/docs/using.md create mode 100644 pages/mkdocs.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 914ab98..8890742 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,12 +4,27 @@ variables: stages: - build - test + - ragdoc_build + - ragdoc_view + - publish + +before_script: + - export GRADLE_USER_HOME=`pwd`/.gradle + +cache: + paths: + - .gradle/wrapper + - .gradle/caches build: image: openjdk:8 stage: build script: - ./gradlew --console=plain --no-daemon assemble + artifacts: + paths: + - "src/gen" + expire_in: 1 week test: image: openjdk:8 @@ -19,3 +34,51 @@ test: artifacts: reports: junit: build/test-results/test/**/TEST-*.xml + +ragdoc_build: + image: + name: "git-st.inf.tu-dresden.de:4567/jastadd/ragdoc-builder" + entrypoint: [""] + stage: ragdoc_build + needs: + - build + script: + - JAVA_FILES=$(find src/ -name '*.java') + - /ragdoc-builder/start-builder.sh -excludeGenerated -d data/ $JAVA_FILES + artifacts: + paths: + - "data/" + +ragdoc_view: + image: + name: "git-st.inf.tu-dresden.de:4567/jastadd/ragdoc-view:relations" + entrypoint: [""] + stage: ragdoc_view + needs: + - ragdoc_build + script: + - DATA_DIR=$(pwd -P)/data + - mkdir -p pages/docs/ragdoc + - OUTPUT_DIR=$(pwd -P)/pages/docs/ragdoc + - cd /ragdoc-view/src/ && rm -rf data && ln -s $DATA_DIR + - /ragdoc-view/build-view.sh --output-path=$OUTPUT_DIR + artifacts: + paths: + - "pages/docs/ragdoc" + +pages: + image: python:3.8-buster + stage: publish + needs: + - ragdoc_view + - test + before_script: + - pip install -U mkdocs mkdocs-macros-plugin mkdocs-git-revision-date-localized-plugin + script: + - cd pages && mkdocs build +# only: +# - develop +# - master + artifacts: + paths: + - public diff --git a/build.gradle b/build.gradle index dfbb25f..cc16a72 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,6 @@ plugins { id 'java' id 'idea' id 'java-test-fixtures' - id 'com.github.ben-manes.versions' version '0.34.0' } ext { diff --git a/pages/.gitignore b/pages/.gitignore new file mode 100644 index 0000000..a95071c --- /dev/null +++ b/pages/.gitignore @@ -0,0 +1 @@ +/docs/ragdoc/ diff --git a/pages/custom_theme/footer.html b/pages/custom_theme/footer.html new file mode 100644 index 0000000..104412a --- /dev/null +++ b/pages/custom_theme/footer.html @@ -0,0 +1,12 @@ +{% block footer %} +<p>{% if config.copyright %} +<small>{{ config.copyright }}<br></small> +{% endif %} +<hr> +Built with <a href="https://www.mkdocs.org/">MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. +{% if page and page.meta and page.meta.git_revision_date_localized %} +<small><br><i>Last updated {{ page.meta.git_revision_date_localized }}</i></small> +{% endif %} +</p> +{% endblock %} + diff --git a/pages/docs/index.md b/pages/docs/index.md new file mode 100644 index 0000000..ad4a5ac --- /dev/null +++ b/pages/docs/index.md @@ -0,0 +1,4 @@ +# Relational RAGs Preprocessor Documentation + +The [Relational RAGs Preprocessor](https://git-st.inf.tu-dresden.de/jastadd/relast-preprocessor) is a framework to create custom preprocessors for [JastAdd](http://jastadd.org) and JastAdd-based [Relational RAGs](http://www.relational-rags.eu). + diff --git a/pages/docs/using.md b/pages/docs/using.md new file mode 100644 index 0000000..a428918 --- /dev/null +++ b/pages/docs/using.md @@ -0,0 +1,200 @@ +# Creating a Preprocessor + +## Project Structure + +The *gradle* build system is required to use the preprocessor. +It is recommended to use the same version of gradle as the preprocessor does internally, which currently is 6.7. + +To create a new preprocessor, follow the following steps: + +1. Create a new gradle project +2. Include this repository as a submodule in the directory `relast.preprocessor` + - copy the contents of this repository into this directory; this can be done using a git submodule + - in your `settings.gradle` add a dependency to it: + ```groovy + include 'relast.preprocessor' + ``` +3. Adapt the `build.gradle` file of your project to use the subproject; this step is described in the next section + +## Constructing a Gradle Build File + +The preprocessor and its test infrastructure are included as project dependencies. + +```groovy +dependencies { + implementation project(':relast.preprocessor') + testImplementation testFixtures(project(":relast.preprocessor")) +} +``` + +Next, the RelAST preprocessor must be called using files provided by the framework and potentially other extensions of the RelAST gramamr DSL. + +```groovy +// Input and output files for relast +def relastInputFiles = [ + "relast.preprocessor/src/main/jastadd/RelAst.relast", + "relast.preprocessor/src/main/jastadd/mustache/Mustache.relast" + // add more files here +] +def relastOutputFiles = [ + "src/gen/jastadd/RelAst.ast", + "src/gen/jastadd/RelAst.jadd", + "src/gen/jastadd/RelAstRefResolver.jadd", + "src/gen/jastadd/RelAstResolverStubs.jrag" +] + +task relast(type: JavaExec) { + classpath = files("relast.preprocessor/libs/relast.jar") + group = 'Build' + + doFirst { + delete relastOutputFiles + mkdir "src/gen/jastadd/" + } + + args = [ + "--listClass=java.util.ArrayList", + "--jastAddList=JastAddList", + "--useJastAddNames", + "--file", + "--resolverHelper", + "--grammarName=./src/gen/jastadd/RelAst" + ] + relastInputFiles + + inputs.files relastInputFiles + outputs.files relastOutputFiles +} +``` + +Finally, JastAdd must be configured to use the correct files. +Note that the scanner includes are split into several files to allow for an extensible lexical analysis. + +```groovy +jastadd { + configureModuleBuild() + modules { + //noinspection GroovyAssignabilityCheck + module("Preprocessor") { + + jastadd { + basedir "." + include "relast.preprocessor/src/main/jastadd/**/*.ast" + include "relast.preprocessor/src/main/jastadd/**/*.jadd" + include "relast.preprocessor/src/main/jastadd/**/*.jrag" + include "src/main/jastadd/**/*.ast" + include "src/main/jastadd/**/*.jadd" + include "src/main/jastadd/**/*.jrag" + include "src/gen/jastadd/**/*.ast" + include "src/gen/jastadd/**/*.jadd" + include "src/gen/jastadd/**/*.jrag" + } + + scanner { + include "relast.preprocessor/src/main/jastadd/scanner/Header.flex", [-4] + include "relast.preprocessor/src/main/jastadd/scanner/Preamble.flex", [-3] + include "relast.preprocessor/src/main/jastadd/scanner/Macros.flex", [-2] + include "relast.preprocessor/src/main/jastadd/scanner/RulesPreamble.flex", [-1] + include "relast.preprocessor/src/main/jastadd/scanner/Keywords.flex", [ 0] + include "relast.preprocessor/src/main/jastadd/scanner/Symbols.flex", [ 1] + include "relast.preprocessor/src/main/jastadd/scanner/RulesPostamble.flex", [ 2] + } + + parser { + include "relast.preprocessor/src/main/jastadd/parser/Preamble.parser" + include "relast.preprocessor/src/main/jastadd/parser/RelAst.parser" + } + } + } + + cleanGen.doFirst { + delete "src/gen" + delete "src/gen-res" + } + + preprocessParser.doFirst { + args += ["--no-beaver-symbol"] + } + + module = "Preprocessor" + astPackage = 'org.jastadd.relast.ast' + parser.name = 'RelAstParser' + genDir = 'src/gen/java' + buildInfoDir = 'src/gen-res' + scanner.genDir = "src/gen/java/org/jastadd/relast/scanner" + parser.genDir = "src/gen/java/org/jastadd/relast/parser" + jastaddOptions = ["--lineColumnNumbers", "--List=JastAddList", "--safeLazy", "--visitCheck=true", "--rewrite=cnta", "--cache=all"] +} +``` + +## Writing Tests + +The preprocessor provides a test framework based on on JUnit 5. +This framework can be used to write acceptance tests, comparing input and output of runs of the constructed preprocessor. +Hereby, no java code has to be written - only a simple configuration file and some input and expected data have to be provided. + +### Test Class + +Create a new class inheriting from `RelAstProcessorTestBase` and set the `mainClass` field to the correct class. + +```java +import org.jastadd.relast.tests.RelAstProcessorTestBase; +import org.junit.jupiter.api.BeforeAll; + +public class MyPreprocessorTest extends RelAstProcessorTestBase { + @BeforeAll + static void init() { + mainClass = Main.class; + } +} +``` + +### Test DSL + +Tests are described by a config file. Each directory in `src/main/resources` containing a `config.yaml` is assumed to contain test data. + +A test configuration must contain a list of objects containing the following properties: + +| Name | Type | Required | Default Value | Description | +| -------------- | --------------------- | -------- | ------------- | ---------------------------------------------------- | +| | | | | | +| `name` | string | yes | -- | name of the test case | +| `args` | list of strings | yes | -- | arguments passed to the preprocessor | +| `fail` | boolean | no | `false` | is the test a negative test | +| | | | | | +| `in` | string | no | `"in"` | directory containing input data | +| `out` | string | no | `"out"` | directory the output is written to | +| `expected` | string | no | `"expected"` | directory containing expected data | +| | | | | | +| `compare` | boolean | no | `false` | should `out` and `expected` directories be compared? | +| | | | | | +| `out-contains` | list of strings | no | empty list | string that must be contained in the `stdout` stream | +| `err-contains` | list of strings | no | empty list | string that must be contained in the `stderr` stream | +| `out-matches` | list of regex strings | no | empty list | regular expression the `stdout` stream must match | +| `err-matches` | list of regex strings | no | empty list | regular expression the `stderr` stream must match | +| | | | | | + +Example: + +```yaml +- name: "SimpleInheritance (null)" + compare: true + out: "null/out" + expected: "null/expected" + args: + - "--inputBaseDir=in" + - "--outputBaseDir=null/out" + - "--printYaml" + - "--errorHandling=null" + - "Example.relast" +- name: "SimpleInheritance (wrong exception)" + fail: true + err-matches: + - "(?s).*Invalid.*" + err-contains: + - "Invalid argument 'not.a.Class' for parameter 'errorHandling': Class not found (name must be qualified)." + args: + - "--inputBaseDir=in" + - "--outputBaseDir=out/does_not_matter_will_fail" + - "--errorHandling=not.a.Class" + - "Example.relast" +``` diff --git a/pages/mkdocs.yml b/pages/mkdocs.yml new file mode 100644 index 0000000..d31d6f9 --- /dev/null +++ b/pages/mkdocs.yml @@ -0,0 +1,18 @@ +site_name: Relational RAGs Preprocessor Documentation +nav: + - using.md + - API documentation: ragdoc/index.html +theme: + name: readthedocs + custom_dir: custom_theme/ +plugins: + - search + - git-revision-date-localized: + type: datetime + timezone: Europe/Berlin + locale: en + fallback_to_build_date: True + - macros +repo_url: https://git-st.inf.tu-dresden.de/jastadd/relast-preprocessor +site_dir: ../public + diff --git a/src/main/resources/preprocessor.properties b/src/main/resources/preprocessor.properties index 7de8ba1..fb55bf0 100644 --- a/src/main/resources/preprocessor.properties +++ b/src/main/resources/preprocessor.properties @@ -1 +1 @@ -version=1.0.0-develop +version=1.0.0-pre-release -- GitLab