Merge branch '9-subtree-update' into 'dev'

Resolve "Subtree update"

See merge request !4

pages/docs/extending.md

@@ -28,5 +28,58 @@ In `ragconnect.base/src/main/java/org/jastadd/ragconnect/compiler/Compiler.java`
 - Set the flag `usesABC` if the choice is given.
 - Add code to add the handler to the list `handlers` if the choice is given, i.e., if `ASTNode.usesABC`
 
-Furthermore, new test cases are appreciated. They can be added in the [ragconnect.rests repository](https://git-st.inf.tu-dresden.de/jastadd/ragconnect-tests)
+Furthermore, new test cases are appreciated, see [below](#writing-tests).
 
+## Writing Tests
+
+To add new tests, have a look at the module `ragconnect.tests`.
+It has three parts:
+1) In `src/test/01-input/*` are the [specifications](#specifications) that are going to be compiled (in principle using the steps described in [the guide to add RagConnect](adding)).
+2) In `src/test/java`, the jUnit 5 [test classes](#test-classes) are implemented. They mostly correspond 1-to-1 to a directory of the first part.
+3) In `build.gradle` the [instructions how to compile](#buildgradle) the specifications using the gradle plugin [PreprocessorPlugin][preprocessor-plugin] (`org.jastadd.preprocessor:testing`).
+
+### Specifications
+
+Every specification must have at least a `README.md` to describe the purpose of the test, a grammar `Test.relast`, and a RagConnect specification `Test.connect`.
+Usually an aspect file `Test.jadd` is included.
+
+### Test Classes
+
+Based on jUnit 5, the test classes testing some behaviour. If sending and/or receiving functionality is used, consider extending `AbstractMqttTest` in order to avoid duplicate code. In case of extending this class, please order the methods according to their lifecycle, i.e.:
+- createModel
+- setupReceiverAndConnect
+- communicateSendInitialValue
+- communicateOnlyUpdatedValue
+- closeConnections
+
+Within `AbstractMqttTest`, an `MqttHandler` named `publisher` is available to publish content.
+Some convenience methods are provided in `TestUtils`, e.g., the `DefaultMappings`, and `mqttUri` to prepend `"mqtt://"` and the correct host for the mqtt broker (`localhost` or a CI-specific host).
+All tests are required to run both locally, and within the CI.
+
+### build.gradle
+
+Use the [PreprocessorPlugin][preprocessor-plugin], the build process can be written concisely in three parts per task:
+
+```groovy
+task compileTreeAllowedTokens(type: RagConnectTest) {
+    ragconnect {
+        outputDir = file('src/test/02-after-ragconnect/treeAllowedTokens')
+        inputFiles = [file('src/test/01-input/treeAllowedTokens/Test.relast'),
+                      file('src/test/01-input/treeAllowedTokens/Test.connect'),
+                      file('src/test/01-input/treeAllowedTokens/TestDependencies.connect')]
+        rootNode = 'Root'
+    }
+    relast {
+        useJastAddNames = true
+        grammarName = 'src/test/03-after-relast/treeAllowedTokens/treeAllowedTokens'
+        serializer = 'jackson'
+    }
+    jastadd {
+        jastAddList = 'JastAddList'
+        packageName = 'treeAllowedTokens.ast'
+        inputFiles = [file('src/test/01-input/treeAllowedTokens/Test.jadd')]
+    }
+}
+```
+
+[preprocessor-plugin]: https://git-st.inf.tu-dresden.de/jastadd/testing

ragconnect.base/build.gradle

@@ -31,7 +31,7 @@ dependencies {
     implementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: "${log4j_version}"
     implementation group: 'org.apache.logging.log4j', name: 'log4j-jul', version: "${log4j_version}"
     implementation group: 'com.github.spullara.mustache.java', name: 'compiler', version: "${mustache_java_version}"
-    runtimeOnly group: 'org.jastadd', name: 'jastadd', version: '2.3.4'
+    runtimeOnly group: 'org.jastadd', name: 'jastadd', version: '2.3.5'
     api group: 'net.sf.beaver', name: 'beaver-rt', version: '0.9.11'
 }
 

ragconnect.base/src/main/jastadd/Analysis.jrag

@@ -86,4 +86,13 @@ aspect Analysis {
 
   // --- needProxyToken ---
   syn boolean TokenComponent.needProxyToken() = !getDependencySourceDefinitionList().isEmpty() || getTokenEndpointDefinitionList().stream().anyMatch(TokenEndpointDefinition::shouldSendValue);
+
+  // --- effectiveUsedAt ---
+  coll Set<EndpointDefinition> MappingDefinition.effectiveUsedAt()
+    [new java.util.HashSet<EndpointDefinition>()]
+    root RagConnect;
+  EndpointDefinition contributes this
+    to MappingDefinition.effectiveUsedAt()
+    for each effectiveMappings();
+
 }

ragconnect.base/src/main/jastadd/Errors.jrag

@@ -32,10 +32,6 @@ aspect Errors {
       when isAlreadyDefined()
       to RagConnect.errors();
 
-    SendTypeEndpointDefinition contributes error("At least one mapping needs to be used")
-      when effectiveMappings().isEmpty()
-      to RagConnect.errors();
-
     DependencyDefinition contributes error("Dependency definition already defined for " + getSource().containingTypeDecl().getName() + " with name " + getID())
       when isAlreadyDefined()
       to RagConnect.errors();
@@ -43,10 +39,6 @@ aspect Errors {
     DependencyDefinition contributes error("The name of a dependency definition must not be equal to a list-node on the source")
       when isAlreadyDefinedAsList()
       to RagConnect.errors();
-
-    DependencyDefinition contributes error("There must be a send endpoint definition targeting " + getSource().parentTypeypeAndName() + " for dependency definition " + getID())
-      when targetEndpointDefinition() == null
-      to RagConnect.errors();
 }
 
 aspect ErrorHelpers {
@@ -58,7 +50,7 @@ aspect ErrorHelpers {
     }
     return false;
   }
-  syn String TokenComponent.parentTypeypeAndName() = containingTypeDecl().getName() + "." + getName();
+  syn String TokenComponent.parentTypeDeclAndName() = containingTypeDecl().getName() + "." + getName();
 }
 
 aspect ErrorMessage {

ragconnect.base/src/main/jastadd/NameResolution.jrag

@@ -27,4 +27,20 @@ aspect NameResolution {
     return null;
   }
 
+  refine RefResolverStubs eq ASTNode.globallyResolveComponentByToken(String id) {
+    // return a Component. id is of the form 'parent_type_name + "." + child_type_name'
+    int dotIndex = id.indexOf(".");
+    String parentTypeName = id.substring(0, dotIndex);
+    String childTypeName = id.substring(dotIndex + 1);
+    TypeDecl type = program().resolveTypeDecl(parentTypeName);
+    // iterate over components and find the matching typeComponent
+    for (Component comp : type.getComponentList()) {
+      if (comp.getName().equals(childTypeName)) {
+        return comp;
+      }
+    }
+    System.err.println("Could not resolve Component '" + id + "'.");
+    return null;
+  }
+
 }

ragconnect.base/src/main/jastadd/Navigation.jrag

@@ -68,12 +68,16 @@ aspect Navigation {
   eq SendTypeEndpointDefinition.asSendTypeEndpointDefinition() = this;
 
   // --- targetEndpointDefinition ---
-  syn SendTokenEndpointDefinition DependencyDefinition.targetEndpointDefinition() {
+  syn EndpointDefinition DependencyDefinition.targetEndpointDefinition() {
     // resolve definition in here, as we do not need resolveMethod in any other place (yet)
     for (EndpointDefinition endpointDefinition : ragconnect().getEndpointDefinitionList()) {
       if (endpointDefinition.isSendTokenEndpointDefinition() &&
           endpointDefinition.asSendTokenEndpointDefinition().getToken().equals(this.getTarget())) {
-        return endpointDefinition.asSendTokenEndpointDefinition();
+        return endpointDefinition;
+      }
+      if (endpointDefinition.isSendTypeEndpointDefinition() &&
+          endpointDefinition.asSendTypeEndpointDefinition().getType().equals(this.getTarget())) {
+        return endpointDefinition;
       }
     }
     return null;

ragconnect.base/src/main/jastadd/RagConnect.relast

@@ -18,7 +18,7 @@ SendTypeEndpointDefinition : TypeEndpointDefinition;
 
 DependencyDefinition ::= <ID>;
 rel DependencyDefinition.Source <-> TokenComponent.DependencySourceDefinition*;
-rel DependencyDefinition.Target -> TokenComponent;
+rel DependencyDefinition.Target -> Component;
 
 MappingDefinition ::= <ID> FromType:MappingDefinitionType <FromVariableName> ToType:MappingDefinitionType <Content> ;
 abstract MappingDefinitionType ::= ;

ragconnect.base/src/main/jastadd/intermediate/Generation.jadd

@@ -22,12 +22,15 @@ aspect AttributesForMustache {
   syn String MEndpointDefinition.firstInputVarName();
   syn String MEndpointDefinition.parentTypeName();
   syn String MEndpointDefinition.entityName();
+  syn String MEndpointDefinition.updateMethod();
+  syn String MEndpointDefinition.writeMethod();
 
   eq MEndpointDefinition.getInnerMappingDefinition(int i).isLast() = i == getNumInnerMappingDefinition() - 1;
   eq MEndpointDefinition.getInnerMappingDefinition(int i).inputVarName() = i == 0 ? firstInputVarName() : getInnerMappingDefinition(i - 1).outputVarName();
 
   syn String MEndpointDefinition.connectParameterName() = "uriString";
   syn String MEndpointDefinition.connectMethod() = "connect" + entityName();
+  syn boolean MEndpointDefinition.isTypeEndpointDefinition() = endpointDef().isTypeEndpointDefinition();
 
   syn String MEndpointDefinition.disconnectMethod() {
     // if both (send and receive) are defined for the token, ensure methods with different names
@@ -54,6 +57,7 @@ aspect AttributesForMustache {
   syn boolean MEndpointDefinition.alwaysApply() = endpointDef().getAlwaysApply();
   syn String MEndpointDefinition.tokenName() = token().getName();
   syn String MEndpointDefinition.typeName() = type().getName();
+  syn String MEndpointDefinition.typeDeclName() = type().getTypeDecl().getName();
   syn MInnerMappingDefinition MEndpointDefinition.lastDefinition() = getInnerMappingDefinition(getNumInnerMappingDefinition() - 1);
   syn String MEndpointDefinition.lastDefinitionToType() = lastDefinition().toType();
   syn String MEndpointDefinition.lastResult() = lastDefinition().outputVarName();
@@ -61,7 +65,7 @@ aspect AttributesForMustache {
     if (lastDefinition().mappingDef().getToType().isArray()) {
       return "java.util.Arrays.equals(" + preemptiveExpectedValue() + ", " + lastResult() + ")";
     }
-    if (token().isPrimitiveType() && lastDefinition().mappingDef().getToType().isPrimitiveType()) {
+    if (endpointDef().isTokenEndpointDefinition() && token().isPrimitiveType() && lastDefinition().mappingDef().getToType().isPrimitiveType()) {
       return preemptiveExpectedValue() + " == " + lastResult();
     }
     if (lastDefinition().mappingDef().getToType().isPrimitiveType() || lastDefinition().mappingDef().isDefaultMappingDefinition()) {
@@ -91,36 +95,40 @@ aspect AttributesForMustache {
   eq MTokenReceiveDefinition.preemptiveReturn() = "return;";
   eq MTokenReceiveDefinition.endpointDef() = getReceiveTokenEndpointDefinition();
   eq MTokenReceiveDefinition.firstInputVarName() = "message";
+  eq MTokenReceiveDefinition.updateMethod() = null;
+  eq MTokenReceiveDefinition.writeMethod() = null;
 
   // --- MTokenSendDefinition ---
   eq MTokenSendDefinition.preemptiveExpectedValue() = lastValue();
   eq MTokenSendDefinition.preemptiveReturn() = "return false;";
   eq MTokenSendDefinition.endpointDef() = getSendTokenEndpointDefinition();
   eq MTokenSendDefinition.firstInputVarName() = "get" + tokenName() + "()";
+  eq MTokenSendDefinition.updateMethod() = "_update_" + tokenName();
+  eq MTokenSendDefinition.writeMethod() = "_writeLastValue_" + tokenName();
 
   syn String MTokenSendDefinition.sender() = "_sender_" + tokenName();
   syn String MTokenSendDefinition.lastValue() = "_lastValue" + tokenName();
-  syn String MTokenSendDefinition.updateMethod() = "_update_" + tokenName();
-  syn String MTokenSendDefinition.writeMethod() = "_writeLastValue_" + tokenName();
   syn String MTokenSendDefinition.tokenResetMethod() = "get" + tokenName() + "_reset";
   syn boolean MTokenSendDefinition.shouldSendValue() = endpointDef().asTokenEndpointDefinition().shouldSendValue();
 
   // MTypeReceiveDefinition
-  eq MTypeReceiveDefinition.preemptiveExpectedValue() = null;
-  eq MTypeReceiveDefinition.preemptiveReturn() = null;
+  eq MTypeReceiveDefinition.preemptiveExpectedValue() = "get" + typeName() + "()";
+  eq MTypeReceiveDefinition.preemptiveReturn() = "return;";
   eq MTypeReceiveDefinition.endpointDef() = getReceiveTypeEndpointDefinition();
-  eq MTypeReceiveDefinition.firstInputVarName() = null;
+  eq MTypeReceiveDefinition.firstInputVarName() = "message";
+  eq MTypeReceiveDefinition.updateMethod() = null;
+  eq MTypeReceiveDefinition.writeMethod() = null;
 
   // MTypeSendDefinition
-  eq MTypeSendDefinition.preemptiveExpectedValue() = null;
-  eq MTypeSendDefinition.preemptiveReturn() = null;
+  eq MTypeSendDefinition.preemptiveExpectedValue() = lastValue();
+  eq MTypeSendDefinition.preemptiveReturn() = "return false;";
   eq MTypeSendDefinition.endpointDef() = getSendTypeEndpointDefinition();
-  eq MTypeSendDefinition.firstInputVarName() = null;
+  eq MTypeSendDefinition.firstInputVarName() = "get" + typeName() + "()";
+  eq MTypeSendDefinition.updateMethod() = "_update_" + typeName();
+  eq MTypeSendDefinition.writeMethod() = "_writeLastValue_" + typeName();
 
   syn String MTypeSendDefinition.sender() = "_sender_" + typeName();
   syn String MTypeSendDefinition.lastValue() = "_lastValue" + typeName();
-  syn String MTypeSendDefinition.updateMethod() = "_update_" + typeName();
-  syn String MTypeSendDefinition.writeMethod() = "_writeLastValue_" + typeName();
   syn String MTypeSendDefinition.tokenResetMethod() = "get" + typeName() + "_reset";
   syn boolean MTypeSendDefinition.shouldSendValue() = endpointDef().asTypeEndpointDefinition().shouldSendValue();
 
@@ -130,13 +138,14 @@ aspect AttributesForMustache {
   syn String MMappingDefinition.fromType() = getMappingDefinition().getFromType().prettyPrint();
   syn String MMappingDefinition.fromVariableName() = getMappingDefinition().getFromVariableName();
   syn String MMappingDefinition.content() = getMappingDefinition().getContent();
+  syn boolean MMappingDefinition.isUsed() = !getMappingDefinition().effectiveUsedAt().isEmpty();
 
   // --- MDependencyDefinition ---
   syn String MDependencyDefinition.targetParentTypeName() = getDependencyDefinition().getTarget().containingTypeDecl().getName();
   syn String MDependencyDefinition.dependencyMethod() = "add" + capitalize(getDependencyDefinition().getID());
   syn String MDependencyDefinition.sourceParentTypeName() = getDependencyDefinition().getSource().containingTypeDecl().getName();
   syn String MDependencyDefinition.internalRelationPrefix() = "_internal_" + getDependencyDefinition().getID();
-  syn nta MTokenSendDefinition MDependencyDefinition.targetEndpointDefinition() {
+  syn nta MEndpointDefinition MDependencyDefinition.targetEndpointDefinition() {
     return getDependencyDefinition().targetEndpointDefinition().toMustache();
   }
 
@@ -208,6 +217,7 @@ aspect AttributesForMustache {
     }
   }
 
+  public abstract MEndpointDefinition EndpointDefinition.toMustache();
   syn lazy MTokenReceiveDefinition ReceiveTokenEndpointDefinition.toMustache() {
     MTokenReceiveDefinition result = new MTokenReceiveDefinition();
     result.setReceiveTokenEndpointDefinition(this);

ragconnect.base/src/main/jastadd/intermediate/Mappings.jrag

@@ -10,9 +10,9 @@ aspect DefaultMappings {
         new JavaMappingDefinitionType(new SimpleJavaTypeUse(typeName));
   }
 
-  private DefaultMappingDefinition RagConnect.baseDefaultMappingDefinition(String fromTypeName, String toTypeName, String content) {
+  private DefaultMappingDefinition RagConnect.createDefaultMappingDefinition(String prefix, String fromTypeName, String toTypeName, String content) {
     DefaultMappingDefinition result = new DefaultMappingDefinition();
-    result.setID("_Default" + baseDefaultMappingTypeNamePart(fromTypeName) + "To" + baseDefaultMappingTypeNamePart(toTypeName) + "Mapping");
+    result.setID(prefix + baseDefaultMappingTypeNamePart(fromTypeName) + "To" + baseDefaultMappingTypeNamePart(toTypeName) + "Mapping");
     result.setFromType(baseDefaultMappingTypeFromName(fromTypeName));
     result.setFromVariableName("input");
     result.setToType(baseDefaultMappingTypeFromName(toTypeName));
@@ -20,6 +20,14 @@ aspect DefaultMappings {
     return result;
   }
 
+  private DefaultMappingDefinition RagConnect.baseDefaultMappingDefinition(String fromTypeName, String toTypeName, String content) {
+    return createDefaultMappingDefinition("_Default", fromTypeName, toTypeName, content);
+  }
+
+  private DefaultMappingDefinition RagConnect.treeDefaultMappingDefinition(String fromTypeName, String toTypeName, String content) {
+    return createDefaultMappingDefinition("_TreeDefault", fromTypeName, toTypeName, content);
+  }
+
   syn nta DefaultMappingDefinition RagConnect.defaultBytesToBooleanMapping() = baseDefaultMappingDefinition(
       "byte[]", "boolean", "return input[0] == (byte) 1;");
   syn nta DefaultMappingDefinition RagConnect.defaultBytesToIntMapping() = baseDefaultMappingDefinition(
@@ -37,6 +45,28 @@ aspect DefaultMappings {
   syn nta DefaultMappingDefinition RagConnect.defaultBytesToStringMapping() = baseDefaultMappingDefinition(
       "byte[]", "String", "return new String(input);");
 
+  syn nta DefaultMappingDefinition RagConnect.defaultBytesToTreeMapping(String typeName) {
+    return treeDefaultMappingDefinition("byte[]", typeName,
+      "String content = new String(input);\n" +
+      "com.fasterxml.jackson.databind.ObjectMapper mapper = new com.fasterxml.jackson.databind.ObjectMapper();\n" +
+      "com.fasterxml.jackson.core.JsonFactory factory = new com.fasterxml.jackson.core.JsonFactory();\n" +
+      "com.fasterxml.jackson.core.JsonParser parser = factory.createParser(content);\n" +
+      typeName + " result = " + typeName + ".deserialize((com.fasterxml.jackson.databind.JsonNode)mapper.readTree(parser));\n" +
+      "parser.close();\n" +
+      "return result;"
+    );
+  }
+  syn nta DefaultMappingDefinition RagConnect.defaultTreeToBytesMapping(String typeName) {
+    return treeDefaultMappingDefinition(typeName, "byte[]",
+      "java.io.ByteArrayOutputStream outputStream = new java.io.ByteArrayOutputStream();\n" +
+      "com.fasterxml.jackson.core.JsonFactory factory = new com.fasterxml.jackson.core.JsonFactory();\n" +
+      "com.fasterxml.jackson.core.JsonGenerator generator = factory.createGenerator(outputStream, com.fasterxml.jackson.core.JsonEncoding.UTF8);\n"+
+      "input.serialize(generator);\n" +
+      "generator.flush();\n" +
+      "return outputStream.toString().getBytes();"
+    );
+  }
+
   syn nta DefaultMappingDefinition RagConnect.defaultBooleanToBytesMapping() = baseDefaultMappingDefinition(
         "boolean", "byte[]", "return java.nio.ByteBuffer.allocate(1).put((byte) (input ? 1 : 0)).array();");
   syn nta DefaultMappingDefinition RagConnect.defaultIntToBytesMapping() = baseDefaultMappingDefinition(
@@ -155,7 +185,13 @@ aspect Mappings {
       case "char":
       case "Character": return ragconnect().defaultBytesToCharMapping();
       case "String": return ragconnect().defaultBytesToStringMapping();
-      default: return null;
+      default:
+        try {
+            TypeDecl typeDecl = program().resolveTypeDecl(targetTypeName());
+            return ragconnect().defaultBytesToTreeMapping(typeDecl.getName());
+        } catch (Exception ignore) {}
+        System.err.println("Could not find suitable default mapping for " + targetTypeName() + " on " + this);
+        return null;
     }
   }
   // --- suitableSendDefaultMapping ---
@@ -176,7 +212,13 @@ aspect Mappings {
       case "char":
       case "Character": return ragconnect().defaultCharToBytesMapping();
       case "String": return ragconnect().defaultStringToBytesMapping();
-      default: return null;
+      default:
+        try {
+            TypeDecl typeDecl = program().resolveTypeDecl(targetTypeName());
+            return ragconnect().defaultTreeToBytesMapping(typeDecl.getName());
+        } catch (Exception ignore) {}
+        System.err.println("Could not find suitable default mapping for " + targetTypeName() + " on " + this);
+        return null;
     }
   }
 
@@ -188,7 +230,9 @@ aspect Mappings {
            getMappingList().get(0).getFromType().prettyPrint();
   }
   eq ReceiveTypeEndpointDefinition.targetTypeName() {
-    return getMappingList().get(0).getFromType().prettyPrint();
+    return getMappingList().isEmpty() ?
+           getType().getTypeDecl().getName() :
+           getMappingList().get(0).getFromType().prettyPrint();
   }
   eq SendTokenEndpointDefinition.targetTypeName() {
     return getMappingList().isEmpty() ?
@@ -196,7 +240,9 @@ aspect Mappings {
            getMappingList().get(getMappingList().size() - 1).getToType().prettyPrint();
   }
   eq SendTypeEndpointDefinition.targetTypeName() {
-    return getMappingList().get(getMappingList().size() - 1).getToType().prettyPrint();
+    return getMappingList().isEmpty() ?
+           getType().getTypeDecl().getName() :
+           getMappingList().get(getMappingList().size() - 1).getToType().prettyPrint();
   }
 
 //  eq ReceiveFromRestDefinition.suitableDefaultMapping() {
@@ -251,8 +297,9 @@ aspect Mappings {
   // --- allMappingDefinitions ---
   syn java.util.List<MappingDefinition> RagConnect.allMappingDefinitions() {
     java.util.List<MappingDefinition> result = new java.util.ArrayList<>();
+    // user-defined mappings
     getMappingDefinitionList().iterator().forEachRemaining(result::add);
-    // byte[] conversion
+    // byte[] <-> primitive conversion
     result.add(defaultBytesToBooleanMapping());
     result.add(defaultBytesToIntMapping());
     result.add(defaultBytesToShortMapping());
@@ -269,6 +316,11 @@ aspect Mappings {
     result.add(defaultDoubleToBytesMapping());
     result.add(defaultCharToBytesMapping());
     result.add(defaultStringToBytesMapping());
+    // byte[] <-> tree conversion
+    for (TypeDecl typeDecl : getProgram().typeDecls()) {
+      result.add(defaultBytesToTreeMapping(typeDecl.getName()));
+      result.add(defaultTreeToBytesMapping(typeDecl.getName()));
+    }
 //    // string conversion
 //    result.add(defaultStringToBooleanMapping());
 //    result.add(defaultStringToIntMapping());

ragconnect.base/src/main/jastadd/parser/RagConnect.parser

@@ -51,11 +51,11 @@ ArrayList string_list
 ;
 
 DependencyDefinition dependency_definition
-  = ID.target_type DOT ID.target_token CAN_DEPEND_ON ID.source_type DOT ID.source_token AS ID.id SCOL
+  = ID.target_type DOT ID.target_component CAN_DEPEND_ON ID.source_type DOT ID.source_token AS ID.id SCOL
     {:
       DependencyDefinition result = new DependencyDefinition();
       result.setSource(TokenComponent.createRef(source_type + "." + source_token));
-      result.setTarget(TokenComponent.createRef(target_type + "." + target_token));
+      result.setTarget(Component.createRef(target_type + "." + target_component));
       result.setID(id);
       return result;
     :}

ragconnect.base/src/main/java/org/jastadd/ragconnect/compiler/Compiler.java

@@ -274,9 +274,10 @@ public class Compiler extends AbstractCompiler {
   }
 
   private void mergeRagConnectDefinitions(RagConnect ragConnect, RagConnect ragConnectToIntegrate) {
-    ragConnect.setEndpointDefinitionList(ragConnectToIntegrate.getEndpointDefinitionList());
-    ragConnect.setMappingDefinitionList(ragConnectToIntegrate.getMappingDefinitionList());
-    ragConnect.setDependencyDefinitionList(ragConnectToIntegrate.getDependencyDefinitionList());
+    ragConnectToIntegrate.getEndpointDefinitionList().forEach(ragConnect::addEndpointDefinition);
+    ragConnectToIntegrate.getMappingDefinitionList().forEach(ragConnect::addMappingDefinition);
+    ragConnectToIntegrate.getDependencyDefinitionList().forEach(ragConnect::addDependencyDefinition);
+    ragConnect.setFileName(ragConnect.getFileName().isEmpty() ? ragConnectToIntegrate.getFileName() : ragConnect.getFileName() + " + " + ragConnectToIntegrate.getFileName());
   }
 
 //  protected void printUsage() {

ragconnect.base/src/main/resources/mappingDefinition.mustache

 protected static {{toType}} ASTNode.{{methodName}}({{fromType}} {{fromVariableName}}) throws Exception {
-  {{{content}}}{{!maybe print line by line to get better indentation}}
+  {{{content}}}
 }

ragconnect.base/src/main/resources/ragconnect.mustache

@@ -18,7 +18,9 @@ aspect RagConnect {
   {{/TypeSendDefinitions}}
 
   {{#MappingDefinitions}}
+  {{#isUsed}}
   {{> mappingDefinition}}
+  {{/isUsed}}
   {{/MappingDefinitions}}
 
   {{#DependencyDefinitions}}
@@ -65,7 +67,9 @@ aspect RagConnectObserver {
       node.trace().setReceiver(this);
     }
     void add(ConnectToken connectToken, ASTNode node, String attributeString, Runnable attributeCall) {
-      System.out.println("** observer add " + node + " on " + attributeString);
+      {{#loggingEnabledForWrites}}
+      System.out.println("** observer add: " + node + " on " + attributeString);
+      {{/loggingEnabledForWrites}}
       observedNodes.add(new RagConnectObserverEntry(connectToken, node, attributeString, attributeCall));
     }
     void remove(ConnectToken connectToken) {
@@ -78,12 +82,16 @@ aspect RagConnectObserver {
       if (event != ASTState.Trace.Event.INC_FLUSH_ATTR) {
         return;
       }
-      System.out.println("** observer check INC_FLUSH_ATTR event");
+      {{#loggingEnabledForWrites}}
+      System.out.println("** observer check INC_FLUSH_ATTR event: " + node + " on " + attribute);
+      {{/loggingEnabledForWrites}}
       // iterate through list, if matching pair. could maybe be more efficient.
       for (RagConnectObserverEntry entry : observedNodes) {
         if (entry.node.equals(node) && entry.attributeString.equals(attribute)) {
           // hit. call the attribute/nta-token
-          System.out.println("** observer hit " + entry.node + " on " + entry.attributeString);
+          {{#loggingEnabledForWrites}}
+          System.out.println("** observer hit: " + entry.node + " on " + entry.attributeString);
+          {{/loggingEnabledForWrites}}
           entry.attributeCall.run();
         }
       }

ragconnect.base/src/main/resources/receiveDefinition.mustache

@@ -3,9 +3,12 @@ public boolean {{parentTypeName}}.{{connectMethod}}(String {{connectParameterNam
   java.util.function.Consumer<byte[]> consumer = message -> {
     {{> mappingApplication}}
     {{#loggingEnabledForReads}}
-    System.out.println("[Receive] " + {{connectParameterName}} + " -> {{tokenName}} = " + {{lastResult}});
+    System.out.println("[Receive] " + {{connectParameterName}} + " -> {{entityName}} = " + {{lastResult}});
     {{/loggingEnabledForReads}}
-    set{{tokenName}}({{lastResult}});
+    {{#isTypeEndpointDefinition}}
+    {{lastResult}}.treeResolveAll();
+    {{/isTypeEndpointDefinition}}
+    set{{entityName}}({{lastResult}});
   };
   ConnectToken connectToken;
   switch (scheme) {

ragconnect.base/src/main/resources/sendDefinition.mustache

@@ -16,7 +16,7 @@ public boolean {{parentTypeName}}.{{connectMethod}}(String {{connectParameterNam
       final String topic = {{mqttHandlerAttribute}}().extractTopic(uri);
       {{sender}} = () -> {
         {{#loggingEnabledForWrites}}
-        System.out.println("[Send] {{tokenName}} = " + get{{tokenName}}() + " -> " + {{connectParameterName}});
+        System.out.println("[Send] {{entityName}} = " + get{{entityName}}() + " -> " + {{connectParameterName}});
         {{/loggingEnabledForWrites}}
         handler.publish(topic, {{lastValue}});
       };
@@ -45,7 +45,7 @@ public boolean {{parentTypeName}}.{{connectMethod}}(String {{connectParameterNam
   connectTokens.computeIfAbsent(this, astNode -> new java.util.HashMap<java.net.URI, ConnectToken>())
                .put(uri, connectToken);
   {{#incrementalOptionActive}}
-  _ragConnectObserver().add(connectToken, this, "{{parentTypeName}}.get{{tokenName}}()", () -> {
+  _ragConnectObserver().add(connectToken, this, "get{{entityName}}", () -> {
     if (this.{{updateMethod}}()) {
       this.{{writeMethod}}();
     }

ragconnect.tests/build.gradle

@@ -36,13 +36,16 @@ repositories {
 dependencies {
     implementation project(':ragconnect.base')
 
-//    runtime group: 'org.jastadd', name: 'jastadd', version: '2.3.4'
-    runtime fileTree(include: ['jastadd2.jar'], dir: '../libs/')
+    runtime group: 'org.jastadd', name: 'jastadd', version: '2.3.5'
 
     testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-api', version: '5.4.0'
     testRuntimeOnly group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.4.0'
     testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.12.1'
 
+    // jackson (for serialization of types)
+    implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.12.1'
+    implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.12.1'
+
     // mqtt
     testImplementation group: 'org.fusesource.mqtt-client', name: 'mqtt-client', version: '1.15'
 
@@ -51,6 +54,7 @@ dependencies {
     testImplementation group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.11.2'
     testImplementation group: 'org.glassfish.jersey.core', name: 'jersey-client', version: '2.31'
     testImplementation group: 'org.glassfish.jersey.inject', name: 'jersey-hk2', version: '2.31'
+    testImplementation group: 'javax.activation', name: 'activation', version: '1.1.1'
 
     testImplementation group: 'net.sf.beaver', name: 'beaver-rt', version: '0.9.11'
     api group: 'com.google.protobuf', name: 'protobuf-java', version: '3.0.0'
@@ -309,3 +313,95 @@ task compileMapping(type: RagConnectTest) {
         inputFiles = [file('src/test/01-input/mapping/Test.jadd')]
     }
 }
+
+// --- Test: tree-manual ---
+task compileTreeManual(type: RagConnectTest, dependsOn: ':ragconnect.base:compileJava') {
+    ragconnect {
+        outputDir = file('src/test/02-after-ragconnect/tree')
+        inputFiles = [file('src/test/01-input/tree/Test.relast'),
+                      file('src/test/01-input/tree/Test.connect'),
+                      file('src/test/01-input/tree/TestDependencies.connect')]
+        rootNode = 'Root'
+    }
+    relast {
+        useJastAddNames = true
+        grammarName = 'src/test/03-after-relast/tree/tree'
+        serializer = 'jackson'
+    }
+    jastadd {
+        jastAddList = 'JastAddList'
+        packageName = 'tree.ast'
+        inputFiles = [file('src/test/01-input/tree/Test.jadd')]
+    }
+}
+
+// --- Test: tree-incremental ---
+task compileTreeIncremental(type: RagConnectTest) {
+    ragconnect {
+        outputDir = file('src/test/02-after-ragconnect/treeInc')
+        inputFiles = [file('src/test/01-input/tree/Test.relast'),
+                      file('src/test/01-input/tree/Test.connect')]
+        rootNode = 'Root'
+    }
+    relast {
+        useJastAddNames = true
+        grammarName = 'src/test/03-after-relast/treeInc/treeInc'
+        serializer = 'jackson'
+    }
+    jastadd {
+        jastAddList = 'JastAddList'
+        packageName = 'treeInc.ast'
+        inputFiles = [file('src/test/01-input/tree/Test.jadd')]
+        extraOptions = ['--tracing=cache,flush',
+                        '--incremental=param',
+                        '--cache=all',
+                        '--rewrite=cnta',
+                        '--flush=full']
+    }
+}
+
+// --- Test: tree-allowed-tokens ---
+task compileTreeAllowedTokens(type: RagConnectTest) {
+    ragconnect {
+        outputDir = file('src/test/02-after-ragconnect/treeAllowedTokens')
+        inputFiles = [file('src/test/01-input/treeAllowedTokens/Test.relast'),
+                      file('src/test/01-input/treeAllowedTokens/Test.connect'),
+                      file('src/test/01-input/treeAllowedTokens/TestDependencies.connect')]
+        rootNode = 'Root'
+    }
+    relast {
+        useJastAddNames = true
+        grammarName = 'src/test/03-after-relast/treeAllowedTokens/treeAllowedTokens'
+        serializer = 'jackson'
+    }
+    jastadd {
+        jastAddList = 'JastAddList'
+        packageName = 'treeAllowedTokens.ast'
+        inputFiles = [file('src/test/01-input/treeAllowedTokens/Test.jadd')]
+    }
+}
+
+// --- Test: tree-allowed-tokens-incremental ---
+task compileTreeAllowedTokensIncremental(type: RagConnectTest) {
+    ragconnect {
+        outputDir = file('src/test/02-after-ragconnect/treeAllowedTokensInc')
+        inputFiles = [file('src/test/01-input/treeAllowedTokens/Test.relast'),
+                      file('src/test/01-input/treeAllowedTokens/Test.connect')]
+        rootNode = 'Root'
+    }
+    relast {
+        useJastAddNames = true
+        grammarName = 'src/test/03-after-relast/treeAllowedTokensInc/treeAllowedTokensInc'
+        serializer = 'jackson'
+    }
+    jastadd {
+        jastAddList = 'JastAddList'
+        packageName = 'treeAllowedTokensInc.ast'
+        inputFiles = [file('src/test/01-input/treeAllowedTokens/Test.jadd')]
+        extraOptions = ['--tracing=cache,flush',
+                        '--incremental=param',
+                        '--cache=all',
+                        '--rewrite=cnta',
+                        '--flush=full']
+    }
+}

ragconnect.tests/src/test/01-input/tree/README.md

+# Tree
+
+Idea: send and receive subtrees, test different relations within the subtree which was sent.
+Once without incremental evaluation (i.e., using manual dependencies), and the other time with incremental evaluation

ragconnect.tests/src/test/01-input/tree/Test.connect

+send tree SenderRoot.Alfa ;
+receive tree ReceiverRoot.Alfa ;

ragconnect.tests/src/test/01-input/tree/Test.jadd

+aspect Computation {
+  syn Alfa SenderRoot.getAlfa() {
+    Alfa result = new Alfa();
+    for (int i = 0; i < 4; i++) {
+      result.addBravo(new Bravo().setID(i));
+      result.addCharlie(new Charlie().setID(i));
+      result.addDelta(new Delta().setID(i));
+    }
+    Bravo inputBravo = result.getBravo(getInput());
+    Charlie inputCharlie = result.getCharlie(getInput());
+    Delta inputDelta = result.getDelta(getInput());
+
+    // rel Alfa.MyBravo -> Bravo ;
+    result.setMyBravo(inputBravo);
+    // rel Alfa.OptionalBravo? -> Bravo ;
+    result.setOptionalBravo(inputBravo);
+    // rel Alfa.MultiBravo* -> Bravo ;
+    result.addMultiBravo(inputBravo);
+    result.addMultiBravo(result.getBravo(getInput() + 1));
+
+    // rel Charlie.MyAlfa? -> Alfa ;
+    inputCharlie.setMyAlfa(result);
+
+    // rel Alfa.SingleBi1Delta <-> Delta.SingleBack1Alfa? ;
+    result.setSingleBi1Delta(inputDelta);
+    // rel Alfa.MultiBi2Delta* <-> Delta.SingleBack2Alfa? ;
+    result.addMultiBi2Delta(inputDelta);
+    result.addMultiBi2Delta(result.getDelta(getInput() + 1));
+    // rel Alfa.MultiBi3Delta* <-> Delta.MultiBack3Alfa* ;
+    result.addMultiBi3Delta(inputDelta);
+    result.addMultiBi3Delta(result.getDelta(getInput() + 1));
+
+    // rel Alfa.Myself -> Alfa ;
+    result.setMyself(result);
+
+    // rel Bravo.MyCharlie -> Charlie ;
+    for (int i = 0; i < 4; i++) {
+      if (i == getInput()) {
+        result.getBravo(i).setMyCharlie(inputCharlie);
+      } else {
+        result.getBravo(i).setMyCharlie(result.getCharlie(0));
+      }
+    }
+    // rel Bravo.OptionalCharlie? -> Charlie ;
+    inputBravo.setOptionalCharlie(inputCharlie);
+    // rel Bravo.MultiCharlie* -> Charlie ;
+    inputBravo.addMultiCharlie(inputCharlie);
+    inputBravo.addMultiCharlie(result.getCharlie(getInput() + 1));
+
+    // rel Bravo.SingleBi1Delta? <-> Delta.SingleBack1Bravo? ;
+    inputBravo.setSingleBi1Delta(inputDelta);
+    // rel Bravo.MultiBi2Delta* <-> Delta.SingleBack2Bravo? ;
+    inputBravo.addMultiBi2Delta(inputDelta);
+    inputBravo.addMultiBi2Delta(result.getDelta(getInput() + 1));
+    // rel Bravo.MultiBi3Delta* <-> Delta.MultiBack3Bravo* ;
+    inputBravo.addMultiBi3Delta(inputDelta);
+    inputBravo.addMultiBi3Delta(result.getDelta(getInput() + 1));
+    result.getBravo(getInput() + 1).addMultiBi3Delta(inputDelta);
+    result.getBravo(getInput() + 1).addMultiBi3Delta(result.getDelta(getInput() + 1));
+
+    return result;
+  }
+
+  syn boolean ASTNode.isNameable() = false;
+  eq Nameable.isNameable() = true;
+}
+
+aspect Testing {
+  class ReceiverRoot implements org.jastadd.ragconnect.tests.AbstractTreeTest.TestWrapperReceiverRoot {}
+  class Alfa implements org.jastadd.ragconnect.tests.AbstractTreeTest.TestWrapperAlfa {}
+  class Bravo implements org.jastadd.ragconnect.tests.AbstractTreeTest.TestWrapperBravo {}
+  class Charlie implements org.jastadd.ragconnect.tests.AbstractTreeTest.TestWrapperCharlie {}
+  class Delta implements org.jastadd.ragconnect.tests.AbstractTreeTest.TestWrapperDelta {}
+}
+
+aspect NameResolution {
+  @Override
+  protected String Nameable.customID() {
+    return getClass().getSimpleName() + getID();
+  }
+}