Version 0.3.2

.gitlab-ci.yml

@@ -52,6 +52,7 @@ publish:
   script:
     - "./gradlew publish"
   only:
+    - dev
     - master
 
 ragdoc_build:
@@ -81,23 +82,40 @@ ragdoc_view:
     - OUTPUT_DIR=$(pwd -P)/pages/docs/ragdoc
     - cd /ragdoc-view/src/ && rm -rf data && ln -s $DATA_DIR
     - /ragdoc-view/build-view.sh --output-path=$OUTPUT_DIR
+  only:
+    - dev
+    - master
   artifacts:
     paths:
       - "pages/docs/ragdoc"
 
 pages:
-  image: python:3.8-buster
+  image: python:3.10.0-bullseye
   stage: publish
   needs:
     - ragdoc_view
     - test
+  variables:
+    PAGES_BRANCH: pages
+    HTTPS_REMOTE: https://${PROJECT_BOT_USER}:${PROJECT_BOT_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git
   before_script:
-    - pip install -U mkdocs mkdocs-macros-plugin mkdocs-git-revision-date-localized-plugin
+    - apt update && apt install git-lfs
+    - pip install -r pages/requirements.txt
+    - git config user.name $PROJECT_BOT_USER
+    - git config user.email $PROJECT_BOT_USER@git-st.inf.tu-dresden.de
+    - git fetch -f origin $PAGES_BRANCH:$PAGES_BRANCH || echo "Pages branch not deployed yet."
+    - git checkout $CI_COMMIT_SHA
   script:
-    - cd pages && mkdocs build
+    - cd pages
+    - export VERSION=$(python main.py)
+    - echo $VERSION
+    - mike list --json --prefix public -r $HTTPS_REMOTE -b $PAGES_BRANCH
+    - mike deploy --rebase --prefix public -r $HTTPS_REMOTE -p -b $PAGES_BRANCH --update-aliases $VERSION
+    - cd ..
+    - git checkout $PAGES_BRANCH -- public/
+  artifacts:
+    paths:
+      - public/
   only:
     - dev
     - master
-  artifacts:
-    paths:
-    - public

pages/docs/adding.md

@@ -51,7 +51,7 @@ You might need to add another task for [compiling relast specifications](#compil
 
 ## Build from source
 
-If you want to use `RagConnect`, the currently suggested way is to first build the jar from the [RagConnect repository](https://git-st.inf.tu-dresden.de/jastadd/ragconnect):
+If you want to plan to extend `RagConnect`, the suggested way is to first build the jar from the [RagConnect repository](https://git-st.inf.tu-dresden.de/jastadd/ragconnect) (if you only want to _use_ it, consider using [the packaged version](#use-packaged-version)).
 
 ```bash
 git clone https://git-st.inf.tu-dresden.de/jastadd/ragconnect.git
@@ -60,14 +60,17 @@ cd ragconnect
 ls ragconnect.base/build/libs/
 ```
 
-This `ragconnect-<version>.jar` can then be copied to your project. Please note, that you can safely use `ragconnect.jar` as filename, because the version can always be printed using `java -jar path/to/ragconnect.jar --version`.
+This `ragconnect-<version>.jar` can then be copied to your project.
+Please note, that you can safely use `ragconnect.jar` as filename, because the version can always be printed using `java -jar path/to/ragconnect.jar --version`.
 
 ```bash
 cp ragconnect.base/build/libs/ragconnect-<version>.jar ../your-project/libs/ragconnect.jar
 cd ../your-project/
 ```
 
-Finally, this jar has to be integrated into your build process. In case, [Gradle](https://gradle.org/) is used, a task could look like the following (example taken from the [ros2rag usecase](https://git-st.inf.tu-dresden.de/jastadd/ros2rag)). The path to the jar file may need to be changed according to your project structure.
+Finally, this jar has to be integrated into your build process.
+In case, [Gradle](https://gradle.org/) is used, a task could look like the following (example taken from the [ros2rag use case](https://git-st.inf.tu-dresden.de/jastadd/ros2rag)).
+The path to the jar file may need to be changed according to your project structure.
 
 ```groovy
 task ragConnect(type: JavaExec) {
@@ -87,9 +90,12 @@ task ragConnect(type: JavaExec) {
 
 You might need to add another task for [compiling relast specifications](#compiling-relast-specifications).
 
-## Compiling RelAst specifications
+# Compiling RelAst specifications
 
-The task to compile `RagConnect` specifications is typically accompanied with a task to invoke the [RelAst compiler](http://relational-rags.eu/) and the [JastAdd gradle plugin](https://plugins.gradle.org/plugin/org.jastadd). The additional arguments `--useJastAddNames`, `--listClass`, `--jastAddList` and `--resolverHelper` to relast are not required. Please see the user manual of the RelAst compiler for more information.
+The task to compile `RagConnect` specifications is typically accompanied by a task to invoke the [RelAst compiler](http://relational-rags.eu/) and the [JastAdd gradle plugin](https://plugins.gradle.org/plugin/org.jastadd).
+Currently, the parameter `--useJastAddNames` is **required**, and it may cause incompatibilities if not set.
+The additional arguments `--listClass`, `--jastAddList` and `--resolverHelper` to relast are not required.
+Please see the user manual of the RelAst compiler for more information.
 
 ```groovy
 task relastToJastAdd(type: JavaExec) {
@@ -118,13 +124,3 @@ One also has to specify the dependencies to get correct ordering of tasks.
 generateAst.dependsOn relastToJastAdd
 relastToJastAdd.dependsOn ragConnect
 ```
-
-## Introduced dependencies
-
-RagConnect itself does not introduce dependencies.
-However, depending on the selected protocols (see [compiler options](using#compiler-options)), additional dependencies are required.
-
-| Protocol | Dependency (Gradle format) | Remarks |
-|-|-|-|
-| `mqtt` | `group: 'org.fusesource.mqtt-client', name: 'mqtt-client', version: '1.15'` | Mqtt is selected by default, so this dependency therefore is required "by default". Might work with other versions as well. |
-| `rest` | `group: 'com.sparkjava', name: 'spark-core', version: '2.9.2'` | Might work with other versions as well. For debugging, it is beneficial to include an implementation for [SLF4J](http://www.slf4j.org/). |

pages/docs/changelog.md

 # Changelog
 
+## 0.3.2
+
+- Allow connection endpoints for list nonterminals ([#21](https://git-st.inf.tu-dresden.de/jastadd/ragconnect/-/issues/21))
+- Ensure correct connect and disconnect functionality ([#31](https://git-st.inf.tu-dresden.de/jastadd/ragconnect/-/issues/31))
+- Enhance documentation ([#13](https://git-st.inf.tu-dresden.de/jastadd/ragconnect/-/issues/13), [#20](https://git-st.inf.tu-dresden.de/jastadd/ragconnect/-/issues/20), [#41](https://git-st.inf.tu-dresden.de/jastadd/ragconnect/-/issues/41))
+
 ## 0.3.1
 
 - Full support for incremental dependency tracking
@@ -15,3 +21,20 @@
 - Add methods to `disconnect` an endpoint
 - Internal: PoC for incremental dependency tracking and subtree endpoint definitions ([#14](https://git-st.inf.tu-dresden.de/jastadd/ragconnect/-/issues/14))
 - Bugfix [#17](https://git-st.inf.tu-dresden.de/jastadd/ragconnect/-/issues/17): Added missing support for `boolean`
+
+## 0.2.2
+
+- Allow normal tokens to be used in send definitions
+
+## 0.2.1
+
+- New communication protocol: REST
+- Selection of protocol when `connect` methods are called, by scheme of given URI
+- Development changes:
+    - Supported printing out YAML data used for mustache templates
+    - Moved string constants to `MRagConnect` structure
+
+## 0.2.0
+
+- Version submitted in paper "A Connection from ROS to RAG-Based Models" (2020)
+- Supported communication protocols: MQTT

pages/docs/compiler.md

+# Compiler options
+
+The compiler is JastAdd-compliant, i.e., it accepts all flags available for JastAdd, though there is no process how to chain pre-processors _yet_.
+Additional options are as follows.
+
+| Name | Required (Default) | Description |
+|-|-|-|
+| `--rootNode` | Yes | Root node in the base grammar. |
+| `--protocols` | No (`mqtt`) | Protocols to enable, currently available: `mqtt, rest`. |
+| `--printYaml` | No (false) | Print out YAML instead of generating files. |
+| `--verbose` | No (false) | Print more messages while compiling. |
+| `--logReads` | No (false) | Enable logging for every received message. |
+| `--logWrites` | No (false) | Enable logging for every sent message. |
+| `--logIncremental` | No (false) | Enable logging for observer in incremental dependency tracking. |
+| `--experimental-jastadd-329` | No (false) | Use trace events `INC_FLUSH_START` and `INC_FLUSH_END` ([JastAdd issue #329][jastadd-issue-329]), see [section about automatic dependency tracking](/using#dependency-tracking-automatically-derived). |
+| `--incremental` | No (false) | Enables incremental dependency tracking (if `trace` is also set appropriately). |
+| `--trace[=flush]` | No (false) | Enables incremental dependency tracking (if `incremental` is also set appropriately). |
+| `--version` | No (false) | Print version info and exit (reused JastAdd option) |
+| `--o` | No (`.`) | Output directory (reused JastAdd option) |
+
+All files to be process have to be passed as arguments.
+Their type is decided by the file extension (`ast` and `relast` for input grammars, `connect` and `ragconnect` for RagConnect definitions file).
+
+# Additional software dependencies
+
+Using RagConnect itself does not introduce dependencies.
+However, depending on the selected protocols and/or used features, additional dependencies are required when using the generated code.
+
+## Communication protocol characteristics
+
+### MQTT
+
+- Protocol identifier: `mqtt`
+- URI scheme: `mqtt://<broker-host>[:port]/<topic>`
+- Default port: 1883
+- Type for mapping definitions: `byte[]`
+- Required runtime dependencies:
+    - `group: 'org.fusesource.mqtt-client', name: 'mqtt-client', version: '1.15'`
+- Additional remarks:
+    - First leading slash not included in topic.
+    - Mqtt is selected by default, so this dependency therefore is required "by default".
+    - Might work with other versions of `org.fusesource.mqtt-client.mqtt.client` as well.
+
+### REST
+
+- Protocol identifier: `rest`
+- URI scheme: `rest://localhost[:port]/<path>`
+- Default port: 4567
+- Type for mapping definitions: `String`
+- Required runtime dependencies:
+    - `group: 'com.sparkjava', name: 'spark-core', version: '2.9.3'`
+- Additional remarks:
+    - Host is always `localhost`.
+    - Might work with newer versions of `com.sparkjava.spark-core` as well.
+    - For debugging, it is beneficial to include an implementation for [SLF4J](http://www.slf4j.org/).
+
+## Used features
+
+### Automatic dependency tracking
+
+- Condition: When passing `--incremental` and `--trace=flush` to RagConnect
+- Required runtime dependencies: _none_
+- Required options for RelAST compiler: _none_
+- Required options for JastAdd:
+    - `--incremental`
+    - `--trace=flush`
+- Remarks:
+    - Other (additional) values passed to those two options must be equal (e.g., `--incremental=param` passed to RagConnect must be also passed to JastAdd)
+    - Other values besides `flush` can be added to `--trace`
+    - [Feature description](/using#dependency-tracking-automatically-derived)
+
+### (Safer) Automatic dependency tracking
+
+- Condition: When passing `--experimental-jastadd-329` to RagConnect
+- Required runtime dependencies: _none_
+- Required options for RelAST compiler: _none_
+- Required options for JastAdd: _none_
+- Remarks:
+    - JastAdd version has to support `INC_FLUSH_START` and `INC_FLUSH_END` (i.e., has [issue #329][jastadd-issue-329] resolved)
+    - [Feature description](/using#dependency-tracking-automatically-derived)
+
+### Tree/List Endpoints
+
+- Condition: When using `tree` or `list` endpoints along with default mappings
+- Required runtime dependencies:
+    - `group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.12.1'`
+    - `group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.12.1'`
+- Required options for RelAST compiler:
+    - `--serializer=jackson`
+- Required options for JastAdd: _none_
+- Remarks:
+    - [Feature description](/using#an-advanced-example)
+
+
+[jastadd-issue-329]: https://bitbucket.org/jastadd/jastadd2/issues/329/add-event-for-completion-of-flush

pages/docs/using.md

@@ -2,14 +2,14 @@
 
 The full example is available at <https://git-st.inf.tu-dresden.de/jastadd/ragconnect-minimal>.
 
-## Preparation
+## Preparation and Specification
 
 The following examples are inspired by the real test case [read1write2](https://git-st.inf.tu-dresden.de/jastadd/ragconnect-tests/-/tree/master/ragconnect.tests/src/test/01-input/read1write2)
 The idea is to have two nonterminals, where input information is received on one of them, and - after transformation - is sent out by both.
 
 Let the following grammar be used:
 
-```bnf
+```
 A ::= <Input:String> /<OutputOnA:String>/ B* ;
 B ::= /<OutputOnB:String>/ ;
 ```
@@ -46,8 +46,11 @@ eq A.getB().input() = getInput();
 ```
 
 In other words, `OutputOnA` depends on `Input` of the same node, and `OutputOnB` depends on `Input` of its parent node.
-Currently, those dependencies have to be explicitely written down. It is expected, that in future version, this won't be necessary anymore.
-This happens also in the DSL (dependencies have to be named to uniquely identify them):
+Currently, those dependencies can be explicitly written down, or incremental evaluation can be used.
+
+### Dependency tracking: Manually specified
+
+This specification happens also in the DSL (dependencies have to be named to uniquely identify them):
 
 ```java
 // dependency definitions
@@ -55,9 +58,19 @@ A.OutputOnA canDependOn A.Input as dependencyA ;
 B.OutputOnB canDependOn A.Input as dependencyB ;
 ```
 
+### Dependency tracking: Automatically derived
+
+To automatically track dependencies, the two additional parameters `--incremental` and `--trace=flush` have to be provided to both RagConnect and (in the later stage) JastAdd.
+This will generate a different implementation of RagConnect relying on enabled incremental evaluation of JastAdd.
+The value for `incremental` has only been tested for `incremental=param`.
+The value for `trace` can include other values besides `flush`.
+
+An experimental, optimized version can be selected using `--experimental-jastadd-329` reducing the risk of conflicts between concurrent attribute evaluations.
+However, this requires a version of JastAdd that resolved the [issue 329](https://bitbucket.org/jastadd/jastadd2/issues/329/add-event-for-completion-of-flush).
+
 ## Using generated code
 
-After specifying everything, code will be generated if [setup properly](adding).
+After specifying everything, code will be generated if [setup properly](/adding).
 Let's create an AST in some driver code:
 
 ```java
@@ -70,7 +83,7 @@ a.addB(b1);
 a.addB(b2);
 ```
 
-Now, we have to set the dependencies as described earlier.
+If necessary, we have to set the dependencies as [described earlier](#dependency-tracking-manually-specified).
 
 ```java
 // a.OutputOnA -> a.Input
@@ -82,7 +95,7 @@ b2.addDependencyB(a);
 ```
 
 Finally, we can actually _connect_ the tokens.
-Depending on the enabled protocols, [different URI schemes are allowed](#communication-protocol-characteristics).
+Depending on the enabled protocols, [different URI schemes are allowed](/compiler#communication-protocol-characteristics).
 In this example, we use the default protocol: MQTT.
 
 ```java
@@ -96,41 +109,16 @@ The first parameter of those connect-methods is always an URI-like String, to id
 In case of MQTT, the server is the host running an MQTT broker, and the path is equal to the topic to publish or subscribe to.
 Please note, that the first leading slash (`/`) is removed for MQTT topics, e.g., for `A.Input` the topic is actually `topic/for/input`.
 
-For sending endpoints, there is a second boolean parameter to specify whether the current value shall be send immediately after connecting.
-
-## Communication protocol characteristics
-
-| Protocol | URI scheme | Default port | Type for mapping definitions | Remarks |
-|-|-|-|-|-|
-| `mqtt` | `mqtt://<broker-host>[:port]/<topic>` | 1883 | `byte[]` | First leading slash not included in topic. |
-| `rest` | `rest://localhost[:port]/<path>` | 4567 | `String` | Host is always `localhost`. |
+For sending endpoints, there is a second boolean parameter to specify whether the current value shall be sent immediately after connecting.
 
-## Compiler options
-
-The compiler is JastAdd-compliant, i.e., it accepts all flags available for JastAdd, though there is no process how to chain pre-processors _yet_.
-Additional options are as follows.
-
-| Name | Required (Default) | Description |
-|-|-|-|
-| `--rootNode` | Yes | Root node in the base grammar. |
-| `--protocols` | No (`mqtt`) | Protocols to enable, currently available: `mqtt, rest`. |
-| `--printYaml` | No (false) | Print out YAML instead of generating files. |
-| `--verbose` | No (false) | Print more messages while compiling. |
-| `--logReads` | No (false) | Enable logging for every received message. |
-| `--logWrites` | No (false) | Enable logging for every sent message. |
-| `--version` | No (false) | Print version info and exit (reused JastAdd option) |
-| `--o` | No (`.`) | Output directory (reused JastAdd option) |
-
-All files to be process have to be passed as arguments. Their type is decided by the file extension (`ast` and `relast` for input grammars, `connect` and `ragconnect` for RagConnect definitions file).
-
-## Remarks
+## Remarks for using manual dependency tracking
 
 When constructing the AST and connecting it, one should always set dependencies before connecting, especially if updates already arriving for receiving endpoints.
 Otherwise, updates might not be propagated after setting dependencies, if values are equal after applying transformations of mapping definitions.
 
 As an example, when using the following grammar and definitions for RagConnect ...
 
-```bnf
+```
 A ::= <Input:int> /<Output:String>/ ;
 ```
 
@@ -146,3 +134,121 @@ Round maps float f to int {:
 ```
 
 ... connecting first could mean to store the first rounded value and not propagating this update, since no dependencies are set, and not propagating further updates leading to the same rounded value even after setting the dependencies.
+
+# An advanced example
+
+Non-terminal children can also be selected as endpoints (not only tokens).
+
+## Normal Non-Terminal Children
+
+Receiving normal non-terminal children and optionals means to replace them with a new node deserialized from the received message.
+Sending them involves serializing a node, and sending this representation in a message.
+
+Suppose, the following (shortened) grammar is used (inspired from the testcase [tree](https://git-st.inf.tu-dresden.de/jastadd/ragconnect-tests/-/tree/master/ragconnect.tests/src/test/01-input/tree))
+
+```
+Root ::= SenderRoot ReceiverRoot ;
+SenderRoot ::= <Input:int> /Alfa/ ;
+ReceiverRoot ::= Alfa ;
+Alfa ::= // some content ...
+```
+
+Now, the complete node of type `Alfa` can be sent, and received again using the following connect specification:
+
+```
+send tree SenderRoot.Alfa ;
+receive tree ReceiverRoot.Alfa ;
+```
+
+Currently, receiving and sending trees requires the explicit demarcation from tokens using the keyword `tree`.
+
+To process non-terminals, default mappings are provided for every non-terminal type of the used grammar.
+They use the JSON serialization offered by the RelAST compiler, i.e., interpret the message as a `String`, deserialize the content reading the message as JSON, or vice versa.
+Additional dependencies are required to use this feature, as detailed in [the compiler section](/compiler#treelist-endpoints).
+
+## Receiving List Children
+
+When receiving list children, there are a few more options to match the connection to given requirements.
+
+Suppose we use a similar grammar as above, i.e.:
+
+```
+SenderRoot ::= /AlfaList:Alfa*/ /SingleAlfa:Alfa/;
+ReceiverRoot ::= Alfa* ;
+```
+
+Several options are possible:
+
+### list
+
+A message for a list endpoint can be interpreted as a complete list (a sequence of nodes of type `Alfa`) by using the `list` keyword instead of `tree`:
+
+```
+receive list ReceiverRoot.Alfa ;
+```
+
+### list + with add
+
+Upon receiving the message, the deserialized list can also be appended to the existing list instead of replace the latter.
+This can be achieved using the keyword `with add` in addition to the keyword `list`:
+
+```
+receive list with add ReceiverRoot.Alfa ;
+```
+
+### tree (indexed)
+
+A message for a list endpoint can also be interpreted as an element of this list.
+
+```
+receive tree ReceiverRoot.Alfa ;
+```
+
+Upon connection, the index of the deserialized element to set, has to be passed (`1` in the example below).
+The list must have enough elements once a message is received.
+
+```java
+receiverRoot.connectAlfa("<some-url>", 1);
+```
+
+### tree (wildcard)
+
+Similar to the `tree (indexed)` case above, messages are interpreted as an element of the list, but the connection can also be made using a "wildcard topic" and without an index.
+Then, once a message is received from a new concrete topic, the deserialized element will be appended to the list and this topic is associated with the index of the newly added element.
+Any further message from that topic will replace the element at the associated index.
+In the short example below, MQTT is used to with a wildcard topic, as `#` matches every sub-topic.
+
+```java
+receiverRoot.connectAlfa("mqtt://<broker>/some/topic/#");
+
+// list is initially empty
+assertEquals(receiverRoot.getAlfaList(), list());
+// after receiving "1" on new topic "some/topic/one" (index 0)
+assertEquals(receiverRoot.getAlfaList(), list("1"));
+// after receiving "other" on new topic "some/topic/two" (index 1)
+assertEquals(receiverRoot.getAlfaList(), list("1", "other"));
+// after receiving "new" on existing topic "some/topic/one" (index 0)
+assertEquals(receiverRoot.getAlfaList(), list("new", "other"));
+```
+
+### tree (indexed/wildcard) + with add
+
+Combining `tree` and `with add` results in a connection, where messages are interpreted as elements of the list, and new elements are appended to the existing list.
+In that case, wildcard and non-wildcard connections behave in the same way, as no index has to be passed, and the element is always append at the end.
+Reusing the example from above, the following observations can be made.
+
+```java
+receiverRoot.connectAlfa("mqtt://<broker>/some/topic/#");
+// or
+receiverRoot.connectAlfa("mqtt://<broker>/some/topic/one");
+receiverRoot.connectAlfa("mqtt://<broker>/some/topic/two");
+
+// list is initially empty
+assertEquals(receiverRoot.getAlfaList(), list());
+// after receiving "1" on topic "some/topic/one"
+assertEquals(receiverRoot.getAlfaList(), list("1"));
+// after receiving "other" on topic "some/topic/two"
+assertEquals(receiverRoot.getAlfaList(), list("1", "other"));
+// after receiving "new" on topic "some/topic/one"
+assertEquals(receiverRoot.getAlfaList(), list("1", "other", "new"));
+```

pages/main.py

+import os
+
 ragconnectVersionFileName = '../ragconnect.base/src/main/resources/ragConnectVersion.properties'
 
 
 def get_version():
+    if os.environ.get('CI_COMMIT_BRANCH', 'unknown') == 'dev':
+        return 'dev'
     with open(ragconnectVersionFileName) as ragconnectVersionFile:
         versionFileContent = ragconnectVersionFile.read()
     return versionFileContent[versionFileContent.rindex('version=') + 8:].strip()

pages/mkdocs.yml

 site_name: RagConnect
+repo_url: https://git-st.inf.tu-dresden.de/jastadd/ragconnect
+site_dir: ../public
+
 nav:
-  - use_cases.md
-  - adding.md
-  - inner-workings.md
-  - using.md
-  - extending.md
-  - changelog.md
-  - API documentation: ragdoc/index.html
+  - "RagConnect by Example": using.md
+  - "Use Cases": use_cases.md
+  - "Adding RagConnect to your project": adding.md
+  - "Compiler options": compiler.md
+  - "Inner workings": inner-workings.md
+  - "Extending RagConnect": extending.md
+  - "Changelog": changelog.md
+  - "API documentation": ragdoc/index.html
+
 theme:
   name: readthedocs
   custom_dir: custom_theme/
+
+markdown_extensions:
+  - toc:
+      permalink: 
+
 plugins:
   - search
   - git-revision-date-localized:
@@ -18,5 +28,3 @@ plugins:
       locale: en
       fallback_to_build_date: True
   - macros
-repo_url: https://git-st.inf.tu-dresden.de/jastadd/ragconnect
-site_dir: ../public

pages/requirements.txt

+mkdocs==1.2.2
+mkdocs-git-revision-date-localized-plugin==0.10.3
+mkdocs-macros-plugin==0.6.3
+mike==1.1.2

ragconnect.base/build.gradle

@@ -34,11 +34,11 @@ dependencies {
 }
 
 def versionFile = 'src/main/resources/ragConnectVersion.properties'
-def oldProps = new Properties()
+def props = new Properties()
 
 try {
-    file(versionFile).withInputStream { stream -> oldProps.load(stream) }
-    version = oldProps['version']
+    file(versionFile).withInputStream { stream -> props.load(stream) }
+    version = props['version']
 } catch (e) {
     // this happens, if either the properties file is not present, or cannot be read from
     throw new GradleException("File ${versionFile} not found or unreadable. Aborting.", e)
@@ -52,9 +52,9 @@ task printVersion() {
 
 task newVersion() {
     doFirst {
-        def props = new Properties()
-        props['version'] = value
-        props.store(file(versionFile).newWriter(), null)
+        def newProps = new Properties()
+        newProps['version'] = value
+        newProps.store(file(versionFile).newWriter(), null)
     }
 }
 
@@ -108,6 +108,14 @@ task relast(type: JavaExec) {
             './src/gen/jastadd/RagConnectResolverStubs.jrag')
 }
 
+clean {
+    delete "src/gen/jastadd/Coverage.jrag"
+    delete "src/gen/jastadd/RagConnect.ast"
+    delete "src/gen/jastadd/RagConnect.jadd"
+    delete "src/gen/jastadd/RagConnectRefResolver.jadd"
+    delete "src/gen/jastadd/RagConnectResolverStubs.jrag"
+}
+
 jastadd {
     configureModuleBuild()
     modules {

ragconnect.base/src/main/jastadd/Analysis.jrag

@@ -95,4 +95,10 @@ aspect Analysis {
     to MappingDefinition.effectiveUsedAt()
     for each effectiveMappings();
 
+  // --- typeIsList ---
+  syn boolean EndpointDefinition.typeIsList() = false;
+  eq TypeEndpointDefinition.typeIsList() {
+    return getType().isListComponent();
+  }
+
 }

ragconnect.base/src/main/jastadd/Configuration.jadd

@@ -3,6 +3,7 @@ aspect Configuration {
   public static boolean ASTNode.loggingEnabledForWrites = false;
   public static boolean ASTNode.loggingEnabledForIncremental = false;
   public static TypeDecl ASTNode.rootNode;
+  public static String ASTNode.JastAddList = "List";
   public static boolean ASTNode.usesMqtt;
   public static boolean ASTNode.usesRest;
   public static boolean ASTNode.incrementalOptionActive;

ragconnect.base/src/main/jastadd/Errors.jrag

-import java.util.Set;
-import java.util.TreeSet;
-import java.util.LinkedList;
-
 aspect Errors {
   coll Set<ErrorMessage> RagConnect.errors()
     [new TreeSet<ErrorMessage>()]

ragconnect.base/src/main/jastadd/Imports.jadd

+import java.util.*;
+
+aspect Imports {
+  // empty
+}

ragconnect.base/src/main/jastadd/Navigation.jrag

-import java.util.List;
-import java.util.ArrayList;
-
 aspect RagConnectNavigation {
 
   // --- program ---

ragconnect.base/src/main/jastadd/RagConnect.relast

@@ -13,10 +13,10 @@ rel TokenEndpointDefinition.Token <-> TokenComponent.TokenEndpointDefinition*;
 ReceiveTokenEndpointDefinition : TokenEndpointDefinition;
 SendTokenEndpointDefinition : TokenEndpointDefinition;
 
-abstract TypeEndpointDefinition : EndpointDefinition;
+abstract TypeEndpointDefinition : EndpointDefinition ::= <UseList:boolean> ;
 rel TypeEndpointDefinition.Type <-> TypeComponent.TypeEndpointDefinition*;
 
-ReceiveTypeEndpointDefinition : TypeEndpointDefinition;
+ReceiveTypeEndpointDefinition : TypeEndpointDefinition ::= <WithAdd:boolean>;
 SendTypeEndpointDefinition : TypeEndpointDefinition;
 
 DependencyDefinition ::= <ID>;

ragconnect.base/src/main/jastadd/intermediate/Generation.jadd

@@ -4,6 +4,9 @@ Design considerations
 */
 
 aspect AttributesForMustache {
+  // --- EndpointDefinition ---
+  syn String EndpointDefinition.idTokenName() = "InternalRagconnectTopicInList";
+
   // --- MRagConnect ---
   eq MRagConnect.getRootTypeComponent(int i).isFirst() = i == 0;
 
@@ -15,6 +18,32 @@ aspect AttributesForMustache {
   syn String MRagConnect.restHandlerAttribute() = "_restHandler";
   syn String MRagConnect.restHandlerField() = "_restHandler";
 
+  syn boolean MRagConnect.hasTreeListEndpoints() = !sendingTreeListEndpoints().isEmpty() || !receivingTreeListEndpoints().isEmpty();
+  syn List<MTypeEndpointDefinition> MRagConnect.sendingTreeListEndpoints() {
+    List<MTypeEndpointDefinition> result = new ArrayList<>();
+    for (var mEndpointDef : getTypeSendDefinitionList()) {
+      if (mEndpointDef.typeIsList()) {
+        result.add(mEndpointDef);
+      }
+    }
+    return result;
+  }
+  syn List<MTypeEndpointDefinition> MRagConnect.receivingTreeListEndpoints() {
+    List<MTypeEndpointDefinition> result = new ArrayList<>();
+    for (var mEndpointDef : getTypeReceiveDefinitionList()) {
+      if (mEndpointDef.typeIsList()) {
+        result.add(mEndpointDef);
+      }
+    }
+    return result;
+  }
+  syn List<TypeDecl> MRagConnect.typesForReceivingListEndpoints() {
+    return receivingTreeListEndpoints().stream()
+      .map(mEndpointDef -> mEndpointDef.type().getTypeDecl())
+      .distinct()
+      .collect(java.util.stream.Collectors.toList());
+  }
+
   // --- MEndpointDefinition ---
   syn String MEndpointDefinition.preemptiveExpectedValue();
   syn String MEndpointDefinition.preemptiveReturn();
@@ -24,16 +53,18 @@ aspect AttributesForMustache {
   syn String MEndpointDefinition.entityName();
   syn String MEndpointDefinition.updateMethod();
   syn String MEndpointDefinition.writeMethod();
+  syn String MEndpointDefinition.getterMethod();
 
   eq MEndpointDefinition.getInnerMappingDefinition(int i).isLast() = i == getNumInnerMappingDefinition() - 1;
   eq MEndpointDefinition.getInnerMappingDefinition(int i).inputVarName() = i == 0 ? firstInputVarName() : getInnerMappingDefinition(i - 1).outputVarName();
 
   syn String MEndpointDefinition.connectParameterName() = "uriString";
   syn String MEndpointDefinition.connectMethod() = "connect" + entityName();
+  syn String MEndpointDefinition.internalConnectMethod() = "_internal_" + connectMethod();
   syn boolean MEndpointDefinition.isTypeEndpointDefinition() = endpointDef().isTypeEndpointDefinition();
 
   syn String MEndpointDefinition.disconnectMethod() {
-    // if both (send and receive) are defined for the token, ensure methods with different names
+    // if both (send and receive) are defined for an endpoint, ensure methods with different names
     String extra;
     if (endpointDef().isTokenEndpointDefinition()) {
       extra = endpointDef().asTokenEndpointDefinition().lookupTokenEndpointDefinitions(token()).size() > 1 ? uniqueSuffix() : "";
@@ -55,6 +86,7 @@ aspect AttributesForMustache {
   syn TokenComponent MEndpointDefinition.token() = endpointDef().asTokenEndpointDefinition().getToken();
   syn TypeComponent MEndpointDefinition.type() = endpointDef().asTypeEndpointDefinition().getType();
   syn boolean MEndpointDefinition.alwaysApply() = endpointDef().getAlwaysApply();
+  syn boolean MEndpointDefinition.typeIsList() = endpointDef().typeIsList();
   syn String MEndpointDefinition.tokenName() = token().getName();
   syn String MEndpointDefinition.typeName() = type().getName();
   syn String MEndpointDefinition.typeDeclName() = type().getTypeDecl().getName();
@@ -69,6 +101,10 @@ aspect AttributesForMustache {
     if (endpointDef().isTokenEndpointDefinition() && token().isPrimitiveType() && lastDefinition().mappingDef().getToType().isPrimitiveType()) {
       return preemptiveExpectedValue() + " == " + lastResult();
     }
+    if (endpointDef().isReceiveTypeEndpointDefinition() && endpointDef().asReceiveTypeEndpointDefinition().getWithAdd()) {
+      // only check if received list is not null
+      return lastResult() + " == null";
+    }
     if (endpointDef().isTypeEndpointDefinition() && type().isOptComponent()) {
       // use "hasX()" instead of "getX() != null" for optionals
       return "has" + typeName() + "()" + " && " + preemptiveExpectedValue() + ".equals(" + lastResult() + ")";
@@ -78,14 +114,20 @@ aspect AttributesForMustache {
     }
     return preemptiveExpectedValue() + " != null ? " + preemptiveExpectedValue() + ".equals(" + lastResult() + ") : " + lastResult() + " == null";
   }
+  syn String MEndpointDefinition.sender() = null; // only for M*SendDefinitions
+  syn String MEndpointDefinition.lastValue() = sender() + ".lastValue"; // only for M*SendDefinitions
 
   // --- MTokenEndpointDefinition ---
+  eq MTokenEndpointDefinition.getterMethod() = "get" + tokenName();
   eq MTokenEndpointDefinition.parentTypeName() = token().containingTypeDecl().getName();
   eq MTokenEndpointDefinition.entityName() = tokenName();
 
   // --- MTypeEndpointDefinition ---
+  syn boolean MTypeEndpointDefinition.isWithAdd() = endpointDef().isReceiveTypeEndpointDefinition() ? endpointDef().asReceiveTypeEndpointDefinition().getWithAdd() : false;
+  syn boolean MTypeEndpointDefinition.isUseList() = endpointDef().asTypeEndpointDefinition().getUseList();
+  eq MTypeEndpointDefinition.getterMethod() = "get" + typeName() + (typeIsList() ? "List" : "");
   eq MTypeEndpointDefinition.parentTypeName() = type().containingTypeDecl().getName();
-  eq MTypeEndpointDefinition.entityName() = typeName();
+  eq MTypeEndpointDefinition.entityName() = typeName() + (isUseList() ? "List" : "");
 
   // --- MInnerMappingDefinition ---
   inh boolean MInnerMappingDefinition.isLast();
@@ -96,7 +138,7 @@ aspect AttributesForMustache {
   syn String MInnerMappingDefinition.outputVarName() = "result" + methodName();  // we do not need "_" in between here, because methodName begins with one
 
   // --- MTokenReceiveDefinition ---
-  eq MTokenReceiveDefinition.preemptiveExpectedValue() = "get" + tokenName() + "()";
+  eq MTokenReceiveDefinition.preemptiveExpectedValue() = getterMethod() + "()";
   eq MTokenReceiveDefinition.preemptiveReturn() = "return;";
   eq MTokenReceiveDefinition.endpointDef() = getReceiveTokenEndpointDefinition();
   eq MTokenReceiveDefinition.firstInputVarName() = "message";
@@ -107,34 +149,35 @@ aspect AttributesForMustache {
   eq MTokenSendDefinition.preemptiveExpectedValue() = lastValue();
   eq MTokenSendDefinition.preemptiveReturn() = "return false;";
   eq MTokenSendDefinition.endpointDef() = getSendTokenEndpointDefinition();
-  eq MTokenSendDefinition.firstInputVarName() = "get" + tokenName() + "()";
+  eq MTokenSendDefinition.firstInputVarName() = getterMethod() + "()";
   eq MTokenSendDefinition.updateMethod() = "_update_" + tokenName();
   eq MTokenSendDefinition.writeMethod() = "_writeLastValue_" + tokenName();
 
-  syn String MTokenSendDefinition.sender() = "_sender_" + tokenName();
-  syn String MTokenSendDefinition.lastValue() = "_lastValue" + tokenName();
-  syn String MTokenSendDefinition.tokenResetMethod() = "get" + tokenName() + "_reset";
+  eq MTokenSendDefinition.sender() = "_sender_" + tokenName();
+  syn String MTokenSendDefinition.tokenResetMethod() = getterMethod() + "_reset";
   syn boolean MTokenSendDefinition.shouldSendValue() = endpointDef().asTokenEndpointDefinition().shouldSendValue();
 
   // MTypeReceiveDefinition
-  eq MTypeReceiveDefinition.preemptiveExpectedValue() = "get" + typeName() + "()";
+  eq MTypeReceiveDefinition.preemptiveExpectedValue() = getterMethod() + "()";
   eq MTypeReceiveDefinition.preemptiveReturn() = "return;";
   eq MTypeReceiveDefinition.endpointDef() = getReceiveTypeEndpointDefinition();
   eq MTypeReceiveDefinition.firstInputVarName() = "message";
   eq MTypeReceiveDefinition.updateMethod() = null;
   eq MTypeReceiveDefinition.writeMethod() = null;
 
+  syn String MTypeReceiveDefinition.resolveInListMethodName() = "_ragconnect_resolve" + entityName() + "InList";
+  syn String MTypeReceiveDefinition.idTokenName() = endpointDef().idTokenName();
+
   // MTypeSendDefinition
   eq MTypeSendDefinition.preemptiveExpectedValue() = lastValue();
   eq MTypeSendDefinition.preemptiveReturn() = "return false;";
   eq MTypeSendDefinition.endpointDef() = getSendTypeEndpointDefinition();
-  eq MTypeSendDefinition.firstInputVarName() = "get" + typeName() + "()";
+  eq MTypeSendDefinition.firstInputVarName() = getterMethod() + "()";
   eq MTypeSendDefinition.updateMethod() = "_update_" + typeName();
   eq MTypeSendDefinition.writeMethod() = "_writeLastValue_" + typeName();
 
-  syn String MTypeSendDefinition.sender() = "_sender_" + typeName();
-  syn String MTypeSendDefinition.lastValue() = "_lastValue" + typeName();
-  syn String MTypeSendDefinition.tokenResetMethod() = "get" + typeName() + "_reset";
+  eq MTypeSendDefinition.sender() = "_sender_" + typeName();
+  syn String MTypeSendDefinition.tokenResetMethod() = getterMethod() + "_reset";
   syn boolean MTypeSendDefinition.shouldSendValue() = endpointDef().asTypeEndpointDefinition().shouldSendValue();
 
   // --- MMappingDefinition ---
@@ -313,7 +356,7 @@ aspect AspectGeneration {
   }
 }
 
-aspect RelationGeneration {
+aspect GrammarGeneration {
   syn java.util.List<Relation> RagConnect.additionalRelations() {
     java.util.List<Relation> result = new java.util.ArrayList<>();
     for (DependencyDefinition dd : allDependencyDefinitionList()) {
@@ -334,6 +377,37 @@ aspect RelationGeneration {
     result.addComment(new WhitespaceComment("\n"));
     return result;
   }
+
+//  coll java.util.Map<TypeDecl, TokenComponent> RagConnect.additionalTokens() [new java.util.HashMap<>()] with put root RagConnect;
+
+//  TypeEndpointDefinition contributes getTokenToCreate()
+//    when typeIsList() && !getUseList()
+//    to RagConnect.additionalTokens()
+////    for ragconnect()
+//    ;
+
+  syn java.util.Map<TypeDecl, TokenComponent> RagConnect.additionalTokens() {
+    java.util.Map<TypeDecl, TokenComponent> result = new java.util.HashMap<>();
+    for (EndpointDefinition def : allEndpointDefinitionList()) {
+      if (def.isTypeEndpointDefinition() && def.getTokenToCreate() != null) {
+        result.put(def.asTypeEndpointDefinition().getType().getTypeDecl(), def.getTokenToCreate());
+      }
+    }
+    return result;
+  }
+
+  syn TokenComponent EndpointDefinition.getTokenToCreate() = null;
+  eq TypeEndpointDefinition.getTokenToCreate() {
+    if (typeIsList() && !getUseList()) {
+      TokenComponent result = new TokenComponent();
+      result.setName(idTokenName());
+      result.setNTA(false);
+      result.setJavaTypeUse(new SimpleJavaTypeUse("String"));
+      return result;
+    } else {
+      return null;
+    }
+  }
 }
 
 aspect GrammarExtension {

ragconnect.base/src/main/jastadd/intermediate/Mappings.jrag

 aspect DefaultMappings {
 
   private String RagConnect.baseDefaultMappingTypeNamePart(String typeName) {
-    return capitalize(typeName).replace("[]", "s");
+    return capitalize(typeName).replace("[]", "s").replace("<", "").replace(">", "List");
   }
 
   private MappingDefinitionType RagConnect.baseDefaultMappingTypeFromName(String typeName) {
@@ -67,6 +67,28 @@ aspect DefaultMappings {
     );
   }
 
+  syn nta DefaultMappingDefinition RagConnect.defaultBytesToListTreeMapping(String typeName) {
+    return treeDefaultMappingDefinition("byte[]", JastAddList + "<" + typeName + ">",
+      "String content = new String(input);\n" +
+      "com.fasterxml.jackson.databind.ObjectMapper mapper = new com.fasterxml.jackson.databind.ObjectMapper();\n" +
+      "com.fasterxml.jackson.core.JsonFactory factory = new com.fasterxml.jackson.core.JsonFactory();\n" +
+      "com.fasterxml.jackson.core.JsonParser parser = factory.createParser(content);\n" +
+      JastAddList + "<" + typeName + ">" + " result = " + typeName + ".deserializeList((com.fasterxml.jackson.databind.node.ArrayNode)mapper.readTree(parser));\n" +
+      "parser.close();\n" +
+      "return result;"
+    );
+  }
+  syn nta DefaultMappingDefinition RagConnect.defaultListTreeToBytesMapping() {
+    return treeDefaultMappingDefinition(JastAddList, "byte[]",
+      "java.io.ByteArrayOutputStream outputStream = new java.io.ByteArrayOutputStream();\n" +
+      "com.fasterxml.jackson.core.JsonFactory factory = new com.fasterxml.jackson.core.JsonFactory();\n" +
+      "com.fasterxml.jackson.core.JsonGenerator generator = factory.createGenerator(outputStream, com.fasterxml.jackson.core.JsonEncoding.UTF8);\n"+
+      "input.serialize(generator);\n" +
+      "generator.flush();\n" +
+      "return outputStream.toString().getBytes();"
+    );
+  }
+
   syn nta DefaultMappingDefinition RagConnect.defaultBooleanToBytesMapping() = baseDefaultMappingDefinition(
         "boolean", "byte[]", "return java.nio.ByteBuffer.allocate(1).put((byte) (input ? 1 : 0)).array();");
   syn nta DefaultMappingDefinition RagConnect.defaultIntToBytesMapping() = baseDefaultMappingDefinition(
@@ -187,13 +209,22 @@ aspect Mappings {
       case "String": return ragconnect().defaultBytesToStringMapping();
       default:
         try {
-            TypeDecl typeDecl = program().resolveTypeDecl(targetTypeName());
-            return ragconnect().defaultBytesToTreeMapping(typeDecl.getName());
+          TypeDecl typeDecl = program().resolveTypeDecl(targetTypeName());
+          // TODO: also support list-types, if list is first type
+          return ragconnect().defaultBytesToTreeMapping(typeDecl.getName());
         } catch (Exception ignore) {}
         System.err.println("Could not find suitable default mapping for " + targetTypeName() + " on " + this);
         return null;
     }
   }
+  eq TypeEndpointDefinition.suitableReceiveDefaultMapping() {
+    try {
+      TypeDecl typeDecl = program().resolveTypeDecl(targetTypeName());
+      return typeIsList() && getUseList() ? ragconnect().defaultBytesToListTreeMapping(typeDecl.getName()) : ragconnect().defaultBytesToTreeMapping(typeDecl.getName());
+    } catch (Exception ignore) {}
+    return super.suitableReceiveDefaultMapping();
+  }
+
   // --- suitableSendDefaultMapping ---
   syn DefaultMappingDefinition EndpointDefinition.suitableSendDefaultMapping() {
     switch (targetTypeName()) {
@@ -214,13 +245,21 @@ aspect Mappings {
       case "String": return ragconnect().defaultStringToBytesMapping();
       default:
         try {
-            TypeDecl typeDecl = program().resolveTypeDecl(targetTypeName());
-            return ragconnect().defaultTreeToBytesMapping(typeDecl.getName());
+          TypeDecl typeDecl = program().resolveTypeDecl(targetTypeName());
+          // TODO: also support list-types, if list is last type
+          return ragconnect().defaultTreeToBytesMapping(typeDecl.getName());
         } catch (Exception ignore) {}
         System.err.println("Could not find suitable default mapping for " + targetTypeName() + " on " + this);
         return null;
     }
   }
+  eq TypeEndpointDefinition.suitableSendDefaultMapping() {
+    try {
+      TypeDecl typeDecl = program().resolveTypeDecl(targetTypeName());
+      return typeIsList() && getUseList() ? ragconnect().defaultListTreeToBytesMapping() : ragconnect().defaultTreeToBytesMapping(typeDecl.getName());
+    } catch (Exception ignore) {}
+    return super.suitableSendDefaultMapping();
+  }
 
   // --- targetTypeName ---
   syn String EndpointDefinition.targetTypeName();
@@ -320,7 +359,9 @@ aspect Mappings {
     for (TypeDecl typeDecl : getProgram().typeDecls()) {
       result.add(defaultBytesToTreeMapping(typeDecl.getName()));
       result.add(defaultTreeToBytesMapping(typeDecl.getName()));
+      result.add(defaultBytesToListTreeMapping(typeDecl.getName()));
     }
+    result.add(defaultListTreeToBytesMapping());
 //    // string conversion
 //    result.add(defaultStringToBooleanMapping());
 //    result.add(defaultStringToIntMapping());

ragconnect.base/src/main/jastadd/intermediate2mustache/MustacheNodesToYAML.jrag

@@ -121,12 +121,14 @@ aspect MustacheNodesToYAML {
 
   syn MappingElement MTypeReceiveDefinition.toYAML() {
     MappingElement result = super.toYAML();
+    result.put("typeIsList", typeIsList());
     result.put("loggingEnabledForReads", loggingEnabledForReads);
     return result;
   }
 
   syn MappingElement MTypeSendDefinition.toYAML() {
     MappingElement result = super.toYAML();
+    result.put("typeIsList", typeIsList());
     result.put("sender", sender());
     result.put("lastValue", lastValue());
     result.put("loggingEnabledForWrites", loggingEnabledForWrites);

ragconnect.base/src/main/jastadd/parser/RagConnect.parser

@@ -46,7 +46,36 @@ EndpointDefinition endpoint_definition_type
   = RECEIVE token_ref           {: return new ReceiveTokenEndpointDefinition().setToken(token_ref); :}
   | SEND token_ref              {: return new SendTokenEndpointDefinition().setToken(token_ref); :}
   | RECEIVE TREE type_ref       {: return new ReceiveTypeEndpointDefinition().setType(type_ref); :}
+  | RECEIVE TREE WITH ADD type_ref
+    {:
+      ReceiveTypeEndpointDefinition result = new ReceiveTypeEndpointDefinition();
+      result.setType(type_ref);
+      result.setWithAdd(true);
+      return result;
+    :}
   | SEND TREE type_ref          {: return new SendTypeEndpointDefinition().setType(type_ref); :}
+  | RECEIVE LIST type_ref
+    {:
+      ReceiveTypeEndpointDefinition result = new ReceiveTypeEndpointDefinition();
+      result.setType(type_ref);
+      result.setUseList(true);
+      return result;
+    :}
+  | RECEIVE LIST WITH ADD type_ref
+    {:
+      ReceiveTypeEndpointDefinition result = new ReceiveTypeEndpointDefinition();
+      result.setType(type_ref);
+      result.setWithAdd(true);
+      result.setUseList(true);
+      return result;
+    :}
+  | SEND LIST type_ref
+    {:
+      SendTypeEndpointDefinition result = new SendTypeEndpointDefinition();
+      result.setType(type_ref);
+      result.setUseList(true);
+      return result;
+    :}
 ;
 
 TokenComponent token_ref

ragconnect.base/src/main/jastadd/scanner/Keywords.flex

@@ -6,3 +6,6 @@
 "to"         { return sym(Terminals.TO); }
 "as"         { return sym(Terminals.AS); }
 "tree"       { return sym(Terminals.TREE); }
+"list"       { return sym(Terminals.LIST); }
+"with"       { return sym(Terminals.WITH); }
+"add"        { return sym(Terminals.ADD); }