Commit 926a6c4b authored by Johannes Mey's avatar Johannes Mey
Browse files

Merge branch 'feature/pages' into 'develop'

Feature/pages

See merge request jastadd/relast-preprocessor!3
parents 6df0df86 3a847a66
......@@ -4,12 +4,27 @@ variables:
stages:
- build
- test
- ragdoc_build
- ragdoc_view
- publish
before_script:
- export GRADLE_USER_HOME=`pwd`/.gradle
cache:
paths:
- .gradle/wrapper
- .gradle/caches
build:
image: openjdk:8
stage: build
script:
- ./gradlew --console=plain --no-daemon assemble
artifacts:
paths:
- "src/gen"
expire_in: 1 week
test:
image: openjdk:8
......@@ -19,3 +34,51 @@ test:
artifacts:
reports:
junit: build/test-results/test/**/TEST-*.xml
ragdoc_build:
image:
name: "git-st.inf.tu-dresden.de:4567/jastadd/ragdoc-builder"
entrypoint: [""]
stage: ragdoc_build
needs:
- build
script:
- JAVA_FILES=$(find src/ -name '*.java')
- /ragdoc-builder/start-builder.sh -excludeGenerated -d data/ $JAVA_FILES
artifacts:
paths:
- "data/"
ragdoc_view:
image:
name: "git-st.inf.tu-dresden.de:4567/jastadd/ragdoc-view:relations"
entrypoint: [""]
stage: ragdoc_view
needs:
- ragdoc_build
script:
- DATA_DIR=$(pwd -P)/data
- mkdir -p pages/docs/ragdoc
- OUTPUT_DIR=$(pwd -P)/pages/docs/ragdoc
- cd /ragdoc-view/src/ && rm -rf data && ln -s $DATA_DIR
- /ragdoc-view/build-view.sh --output-path=$OUTPUT_DIR
artifacts:
paths:
- "pages/docs/ragdoc"
pages:
image: python:3.8-buster
stage: publish
needs:
- ragdoc_view
- test
before_script:
- pip install -U mkdocs mkdocs-macros-plugin mkdocs-git-revision-date-localized-plugin
script:
- cd pages && mkdocs build
# only:
# - develop
# - master
artifacts:
paths:
- public
......@@ -5,7 +5,6 @@ plugins {
id 'java'
id 'idea'
id 'java-test-fixtures'
id 'com.github.ben-manes.versions' version '0.34.0'
}
ext {
......
/docs/ragdoc/
{% block footer %}
<p>{% if config.copyright %}
<small>{{ config.copyright }}<br></small>
{% endif %}
<hr>
Built with <a href="https://www.mkdocs.org/">MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
{% if page and page.meta and page.meta.git_revision_date_localized %}
<small><br><i>Last updated {{ page.meta.git_revision_date_localized }}</i></small>
{% endif %}
</p>
{% endblock %}
# Relational RAGs Preprocessor Documentation
The [Relational RAGs Preprocessor](https://git-st.inf.tu-dresden.de/jastadd/relast-preprocessor) is a framework to create custom preprocessors for [JastAdd](http://jastadd.org) and JastAdd-based [Relational RAGs](http://www.relational-rags.eu).
# Creating a Preprocessor
## Project Structure
The *gradle* build system is required to use the preprocessor.
It is recommended to use the same version of gradle as the preprocessor does internally, which currently is 6.7.
To create a new preprocessor, follow the following steps:
1. Create a new gradle project
2. Include this repository as a submodule in the directory `relast.preprocessor`
- copy the contents of this repository into this directory; this can be done using a git submodule
- in your `settings.gradle` add a dependency to it:
```groovy
include 'relast.preprocessor'
```
3. Adapt the `build.gradle` file of your project to use the subproject; this step is described in the next section
## Constructing a Gradle Build File
The preprocessor and its test infrastructure are included as project dependencies.
```groovy
dependencies {
implementation project(':relast.preprocessor')
testImplementation testFixtures(project(":relast.preprocessor"))
}
```
Next, the RelAST preprocessor must be called using files provided by the framework and potentially other extensions of the RelAST gramamr DSL.
```groovy
// Input and output files for relast
def relastInputFiles = [
"relast.preprocessor/src/main/jastadd/RelAst.relast",
"relast.preprocessor/src/main/jastadd/mustache/Mustache.relast"
// add more files here
]
def relastOutputFiles = [
"src/gen/jastadd/RelAst.ast",
"src/gen/jastadd/RelAst.jadd",
"src/gen/jastadd/RelAstRefResolver.jadd",
"src/gen/jastadd/RelAstResolverStubs.jrag"
]
task relast(type: JavaExec) {
classpath = files("relast.preprocessor/libs/relast.jar")
group = 'Build'
doFirst {
delete relastOutputFiles
mkdir "src/gen/jastadd/"
}
args = [
"--listClass=java.util.ArrayList",
"--jastAddList=JastAddList",
"--useJastAddNames",
"--file",
"--resolverHelper",
"--grammarName=./src/gen/jastadd/RelAst"
] + relastInputFiles
inputs.files relastInputFiles
outputs.files relastOutputFiles
}
```
Finally, JastAdd must be configured to use the correct files.
Note that the scanner includes are split into several files to allow for an extensible lexical analysis.
```groovy
jastadd {
configureModuleBuild()
modules {
//noinspection GroovyAssignabilityCheck
module("Preprocessor") {
jastadd {
basedir "."
include "relast.preprocessor/src/main/jastadd/**/*.ast"
include "relast.preprocessor/src/main/jastadd/**/*.jadd"
include "relast.preprocessor/src/main/jastadd/**/*.jrag"
include "src/main/jastadd/**/*.ast"
include "src/main/jastadd/**/*.jadd"
include "src/main/jastadd/**/*.jrag"
include "src/gen/jastadd/**/*.ast"
include "src/gen/jastadd/**/*.jadd"
include "src/gen/jastadd/**/*.jrag"
}
scanner {
include "relast.preprocessor/src/main/jastadd/scanner/Header.flex", [-4]
include "relast.preprocessor/src/main/jastadd/scanner/Preamble.flex", [-3]
include "relast.preprocessor/src/main/jastadd/scanner/Macros.flex", [-2]
include "relast.preprocessor/src/main/jastadd/scanner/RulesPreamble.flex", [-1]
include "relast.preprocessor/src/main/jastadd/scanner/Keywords.flex", [ 0]
include "relast.preprocessor/src/main/jastadd/scanner/Symbols.flex", [ 1]
include "relast.preprocessor/src/main/jastadd/scanner/RulesPostamble.flex", [ 2]
}
parser {
include "relast.preprocessor/src/main/jastadd/parser/Preamble.parser"
include "relast.preprocessor/src/main/jastadd/parser/RelAst.parser"
}
}
}
cleanGen.doFirst {
delete "src/gen"
delete "src/gen-res"
}
preprocessParser.doFirst {
args += ["--no-beaver-symbol"]
}
module = "Preprocessor"
astPackage = 'org.jastadd.relast.ast'
parser.name = 'RelAstParser'
genDir = 'src/gen/java'
buildInfoDir = 'src/gen-res'
scanner.genDir = "src/gen/java/org/jastadd/relast/scanner"
parser.genDir = "src/gen/java/org/jastadd/relast/parser"
jastaddOptions = ["--lineColumnNumbers", "--List=JastAddList", "--safeLazy", "--visitCheck=true", "--rewrite=cnta", "--cache=all"]
}
```
## Writing Tests
The preprocessor provides a test framework based on on JUnit 5.
This framework can be used to write acceptance tests, comparing input and output of runs of the constructed preprocessor.
Hereby, no java code has to be written - only a simple configuration file and some input and expected data have to be provided.
### Test Class
Create a new class inheriting from `RelAstProcessorTestBase` and set the `mainClass` field to the correct class.
```java
import org.jastadd.relast.tests.RelAstProcessorTestBase;
import org.junit.jupiter.api.BeforeAll;
public class MyPreprocessorTest extends RelAstProcessorTestBase {
@BeforeAll
static void init() {
mainClass = Main.class;
}
}
```
### Test DSL
Tests are described by a config file. Each directory in `src/main/resources` containing a `config.yaml` is assumed to contain test data.
A test configuration must contain a list of objects containing the following properties:
| Name | Type | Required | Default Value | Description |
| -------------- | --------------------- | -------- | ------------- | ---------------------------------------------------- |
| | | | | |
| `name` | string | yes | -- | name of the test case |
| `args` | list of strings | yes | -- | arguments passed to the preprocessor |
| `fail` | boolean | no | `false` | is the test a negative test |
| | | | | |
| `in` | string | no | `"in"` | directory containing input data |
| `out` | string | no | `"out"` | directory the output is written to |
| `expected` | string | no | `"expected"` | directory containing expected data |
| | | | | |
| `compare` | boolean | no | `false` | should `out` and `expected` directories be compared? |
| | | | | |
| `out-contains` | list of strings | no | empty list | string that must be contained in the `stdout` stream |
| `err-contains` | list of strings | no | empty list | string that must be contained in the `stderr` stream |
| `out-matches` | list of regex strings | no | empty list | regular expression the `stdout` stream must match |
| `err-matches` | list of regex strings | no | empty list | regular expression the `stderr` stream must match |
| | | | | |
Example:
```yaml
- name: "SimpleInheritance (null)"
compare: true
out: "null/out"
expected: "null/expected"
args:
- "--inputBaseDir=in"
- "--outputBaseDir=null/out"
- "--printYaml"
- "--errorHandling=null"
- "Example.relast"
- name: "SimpleInheritance (wrong exception)"
fail: true
err-matches:
- "(?s).*Invalid.*"
err-contains:
- "Invalid argument 'not.a.Class' for parameter 'errorHandling': Class not found (name must be qualified)."
args:
- "--inputBaseDir=in"
- "--outputBaseDir=out/does_not_matter_will_fail"
- "--errorHandling=not.a.Class"
- "Example.relast"
```
site_name: Relational RAGs Preprocessor Documentation
nav:
- using.md
- API documentation: ragdoc/index.html
theme:
name: readthedocs
custom_dir: custom_theme/
plugins:
- search
- git-revision-date-localized:
type: datetime
timezone: Europe/Berlin
locale: en
fallback_to_build_date: True
- macros
repo_url: https://git-st.inf.tu-dresden.de/jastadd/relast-preprocessor
site_dir: ../public
version=1.0.0-develop
version=1.0.0-pre-release
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment