From 3a847a6612ed71252c0f11dc245328b730695a55 Mon Sep 17 00:00:00 2001
From: Johannes Mey <johannes.mey@tu-dresden.de>
Date: Sun, 7 Feb 2021 12:18:12 +0000
Subject: [PATCH] Feature/pages
---
.gitlab-ci.yml | 63 +++++++
build.gradle | 1 -
pages/.gitignore | 1 +
pages/custom_theme/footer.html | 12 ++
pages/docs/index.md | 4 +
pages/docs/using.md | 200 +++++++++++++++++++++
pages/mkdocs.yml | 18 ++
src/main/resources/preprocessor.properties | 2 +-
8 files changed, 299 insertions(+), 2 deletions(-)
create mode 100644 pages/.gitignore
create mode 100644 pages/custom_theme/footer.html
create mode 100644 pages/docs/index.md
create mode 100644 pages/docs/using.md
create mode 100644 pages/mkdocs.yml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 914ab98..8890742 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,12 +4,27 @@ variables:
stages:
- build
- test
+ - ragdoc_build
+ - ragdoc_view
+ - publish
+
+before_script:
+ - export GRADLE_USER_HOME=`pwd`/.gradle
+
+cache:
+ paths:
+ - .gradle/wrapper
+ - .gradle/caches
build:
image: openjdk:8
stage: build
script:
- ./gradlew --console=plain --no-daemon assemble
+ artifacts:
+ paths:
+ - "src/gen"
+ expire_in: 1 week
test:
image: openjdk:8
@@ -19,3 +34,51 @@ test:
artifacts:
reports:
junit: build/test-results/test/**/TEST-*.xml
+
+ragdoc_build:
+ image:
+ name: "git-st.inf.tu-dresden.de:4567/jastadd/ragdoc-builder"
+ entrypoint: [""]
+ stage: ragdoc_build
+ needs:
+ - build
+ script:
+ - JAVA_FILES=$(find src/ -name '*.java')
+ - /ragdoc-builder/start-builder.sh -excludeGenerated -d data/ $JAVA_FILES
+ artifacts:
+ paths:
+ - "data/"
+
+ragdoc_view:
+ image:
+ name: "git-st.inf.tu-dresden.de:4567/jastadd/ragdoc-view:relations"
+ entrypoint: [""]
+ stage: ragdoc_view
+ needs:
+ - ragdoc_build
+ script:
+ - DATA_DIR=$(pwd -P)/data
+ - mkdir -p pages/docs/ragdoc
+ - OUTPUT_DIR=$(pwd -P)/pages/docs/ragdoc
+ - cd /ragdoc-view/src/ && rm -rf data && ln -s $DATA_DIR
+ - /ragdoc-view/build-view.sh --output-path=$OUTPUT_DIR
+ artifacts:
+ paths:
+ - "pages/docs/ragdoc"
+
+pages:
+ image: python:3.8-buster
+ stage: publish
+ needs:
+ - ragdoc_view
+ - test
+ before_script:
+ - pip install -U mkdocs mkdocs-macros-plugin mkdocs-git-revision-date-localized-plugin
+ script:
+ - cd pages && mkdocs build
+# only:
+# - develop
+# - master
+ artifacts:
+ paths:
+ - public
diff --git a/build.gradle b/build.gradle
index dfbb25f..cc16a72 100644
--- a/build.gradle
+++ b/build.gradle
@@ -5,7 +5,6 @@ plugins {
id 'java'
id 'idea'
id 'java-test-fixtures'
- id 'com.github.ben-manes.versions' version '0.34.0'
}
ext {
diff --git a/pages/.gitignore b/pages/.gitignore
new file mode 100644
index 0000000..a95071c
--- /dev/null
+++ b/pages/.gitignore
@@ -0,0 +1 @@
+/docs/ragdoc/
diff --git a/pages/custom_theme/footer.html b/pages/custom_theme/footer.html
new file mode 100644
index 0000000..104412a
--- /dev/null
+++ b/pages/custom_theme/footer.html
@@ -0,0 +1,12 @@
+{% block footer %}
+<p>{% if config.copyright %}
+<small>{{ config.copyright }}<br></small>
+{% endif %}
+<hr>
+Built with <a href="https://www.mkdocs.org/">MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+{% if page and page.meta and page.meta.git_revision_date_localized %}
+<small><br><i>Last updated {{ page.meta.git_revision_date_localized }}</i></small>
+{% endif %}
+</p>
+{% endblock %}
+
diff --git a/pages/docs/index.md b/pages/docs/index.md
new file mode 100644
index 0000000..ad4a5ac
--- /dev/null
+++ b/pages/docs/index.md
@@ -0,0 +1,4 @@
+# Relational RAGs Preprocessor Documentation
+
+The [Relational RAGs Preprocessor](https://git-st.inf.tu-dresden.de/jastadd/relast-preprocessor) is a framework to create custom preprocessors for [JastAdd](http://jastadd.org) and JastAdd-based [Relational RAGs](http://www.relational-rags.eu).
+
diff --git a/pages/docs/using.md b/pages/docs/using.md
new file mode 100644
index 0000000..a428918
--- /dev/null
+++ b/pages/docs/using.md
@@ -0,0 +1,200 @@
+# Creating a Preprocessor
+
+## Project Structure
+
+The *gradle* build system is required to use the preprocessor.
+It is recommended to use the same version of gradle as the preprocessor does internally, which currently is 6.7.
+
+To create a new preprocessor, follow the following steps:
+
+1. Create a new gradle project
+2. Include this repository as a submodule in the directory `relast.preprocessor`
+ - copy the contents of this repository into this directory; this can be done using a git submodule
+ - in your `settings.gradle` add a dependency to it:
+ ```groovy
+ include 'relast.preprocessor'
+ ```
+3. Adapt the `build.gradle` file of your project to use the subproject; this step is described in the next section
+
+## Constructing a Gradle Build File
+
+The preprocessor and its test infrastructure are included as project dependencies.
+
+```groovy
+dependencies {
+ implementation project(':relast.preprocessor')
+ testImplementation testFixtures(project(":relast.preprocessor"))
+}
+```
+
+Next, the RelAST preprocessor must be called using files provided by the framework and potentially other extensions of the RelAST gramamr DSL.
+
+```groovy
+// Input and output files for relast
+def relastInputFiles = [
+ "relast.preprocessor/src/main/jastadd/RelAst.relast",
+ "relast.preprocessor/src/main/jastadd/mustache/Mustache.relast"
+ // add more files here
+]
+def relastOutputFiles = [
+ "src/gen/jastadd/RelAst.ast",
+ "src/gen/jastadd/RelAst.jadd",
+ "src/gen/jastadd/RelAstRefResolver.jadd",
+ "src/gen/jastadd/RelAstResolverStubs.jrag"
+]
+
+task relast(type: JavaExec) {
+ classpath = files("relast.preprocessor/libs/relast.jar")
+ group = 'Build'
+
+ doFirst {
+ delete relastOutputFiles
+ mkdir "src/gen/jastadd/"
+ }
+
+ args = [
+ "--listClass=java.util.ArrayList",
+ "--jastAddList=JastAddList",
+ "--useJastAddNames",
+ "--file",
+ "--resolverHelper",
+ "--grammarName=./src/gen/jastadd/RelAst"
+ ] + relastInputFiles
+
+ inputs.files relastInputFiles
+ outputs.files relastOutputFiles
+}
+```
+
+Finally, JastAdd must be configured to use the correct files.
+Note that the scanner includes are split into several files to allow for an extensible lexical analysis.
+
+```groovy
+jastadd {
+ configureModuleBuild()
+ modules {
+ //noinspection GroovyAssignabilityCheck
+ module("Preprocessor") {
+
+ jastadd {
+ basedir "."
+ include "relast.preprocessor/src/main/jastadd/**/*.ast"
+ include "relast.preprocessor/src/main/jastadd/**/*.jadd"
+ include "relast.preprocessor/src/main/jastadd/**/*.jrag"
+ include "src/main/jastadd/**/*.ast"
+ include "src/main/jastadd/**/*.jadd"
+ include "src/main/jastadd/**/*.jrag"
+ include "src/gen/jastadd/**/*.ast"
+ include "src/gen/jastadd/**/*.jadd"
+ include "src/gen/jastadd/**/*.jrag"
+ }
+
+ scanner {
+ include "relast.preprocessor/src/main/jastadd/scanner/Header.flex", [-4]
+ include "relast.preprocessor/src/main/jastadd/scanner/Preamble.flex", [-3]
+ include "relast.preprocessor/src/main/jastadd/scanner/Macros.flex", [-2]
+ include "relast.preprocessor/src/main/jastadd/scanner/RulesPreamble.flex", [-1]
+ include "relast.preprocessor/src/main/jastadd/scanner/Keywords.flex", [ 0]
+ include "relast.preprocessor/src/main/jastadd/scanner/Symbols.flex", [ 1]
+ include "relast.preprocessor/src/main/jastadd/scanner/RulesPostamble.flex", [ 2]
+ }
+
+ parser {
+ include "relast.preprocessor/src/main/jastadd/parser/Preamble.parser"
+ include "relast.preprocessor/src/main/jastadd/parser/RelAst.parser"
+ }
+ }
+ }
+
+ cleanGen.doFirst {
+ delete "src/gen"
+ delete "src/gen-res"
+ }
+
+ preprocessParser.doFirst {
+ args += ["--no-beaver-symbol"]
+ }
+
+ module = "Preprocessor"
+ astPackage = 'org.jastadd.relast.ast'
+ parser.name = 'RelAstParser'
+ genDir = 'src/gen/java'
+ buildInfoDir = 'src/gen-res'
+ scanner.genDir = "src/gen/java/org/jastadd/relast/scanner"
+ parser.genDir = "src/gen/java/org/jastadd/relast/parser"
+ jastaddOptions = ["--lineColumnNumbers", "--List=JastAddList", "--safeLazy", "--visitCheck=true", "--rewrite=cnta", "--cache=all"]
+}
+```
+
+## Writing Tests
+
+The preprocessor provides a test framework based on on JUnit 5.
+This framework can be used to write acceptance tests, comparing input and output of runs of the constructed preprocessor.
+Hereby, no java code has to be written - only a simple configuration file and some input and expected data have to be provided.
+
+### Test Class
+
+Create a new class inheriting from `RelAstProcessorTestBase` and set the `mainClass` field to the correct class.
+
+```java
+import org.jastadd.relast.tests.RelAstProcessorTestBase;
+import org.junit.jupiter.api.BeforeAll;
+
+public class MyPreprocessorTest extends RelAstProcessorTestBase {
+ @BeforeAll
+ static void init() {
+ mainClass = Main.class;
+ }
+}
+```
+
+### Test DSL
+
+Tests are described by a config file. Each directory in `src/main/resources` containing a `config.yaml` is assumed to contain test data.
+
+A test configuration must contain a list of objects containing the following properties:
+
+| Name | Type | Required | Default Value | Description |
+| -------------- | --------------------- | -------- | ------------- | ---------------------------------------------------- |
+| | | | | |
+| `name` | string | yes | -- | name of the test case |
+| `args` | list of strings | yes | -- | arguments passed to the preprocessor |
+| `fail` | boolean | no | `false` | is the test a negative test |
+| | | | | |
+| `in` | string | no | `"in"` | directory containing input data |
+| `out` | string | no | `"out"` | directory the output is written to |
+| `expected` | string | no | `"expected"` | directory containing expected data |
+| | | | | |
+| `compare` | boolean | no | `false` | should `out` and `expected` directories be compared? |
+| | | | | |
+| `out-contains` | list of strings | no | empty list | string that must be contained in the `stdout` stream |
+| `err-contains` | list of strings | no | empty list | string that must be contained in the `stderr` stream |
+| `out-matches` | list of regex strings | no | empty list | regular expression the `stdout` stream must match |
+| `err-matches` | list of regex strings | no | empty list | regular expression the `stderr` stream must match |
+| | | | | |
+
+Example:
+
+```yaml
+- name: "SimpleInheritance (null)"
+ compare: true
+ out: "null/out"
+ expected: "null/expected"
+ args:
+ - "--inputBaseDir=in"
+ - "--outputBaseDir=null/out"
+ - "--printYaml"
+ - "--errorHandling=null"
+ - "Example.relast"
+- name: "SimpleInheritance (wrong exception)"
+ fail: true
+ err-matches:
+ - "(?s).*Invalid.*"
+ err-contains:
+ - "Invalid argument 'not.a.Class' for parameter 'errorHandling': Class not found (name must be qualified)."
+ args:
+ - "--inputBaseDir=in"
+ - "--outputBaseDir=out/does_not_matter_will_fail"
+ - "--errorHandling=not.a.Class"
+ - "Example.relast"
+```
diff --git a/pages/mkdocs.yml b/pages/mkdocs.yml
new file mode 100644
index 0000000..d31d6f9
--- /dev/null
+++ b/pages/mkdocs.yml
@@ -0,0 +1,18 @@
+site_name: Relational RAGs Preprocessor Documentation
+nav:
+ - using.md
+ - API documentation: ragdoc/index.html
+theme:
+ name: readthedocs
+ custom_dir: custom_theme/
+plugins:
+ - search
+ - git-revision-date-localized:
+ type: datetime
+ timezone: Europe/Berlin
+ locale: en
+ fallback_to_build_date: True
+ - macros
+repo_url: https://git-st.inf.tu-dresden.de/jastadd/relast-preprocessor
+site_dir: ../public
+
diff --git a/src/main/resources/preprocessor.properties b/src/main/resources/preprocessor.properties
index 7de8ba1..fb55bf0 100644
--- a/src/main/resources/preprocessor.properties
+++ b/src/main/resources/preprocessor.properties
@@ -1 +1 @@
-version=1.0.0-develop
+version=1.0.0-pre-release
--
GitLab