Merge branch 'dev' into 'master'

Updating master with current development state.

Closes #20, #14, #18, and #15

See merge request !4

.gitignore

 .gradle/
 .idea/
 build/dependencyUpdates/
+venv/
+*/out/
+logs/

.gitlab-ci.yml

@@ -4,15 +4,23 @@ stages:
 - report
 
 variables:
+  # Instruct Testcontainers to use the daemon of DinD.
+  DOCKER_HOST: "unix:///var/run/docker.sock"
+#  # Improve performance with overlayfs.
+#  DOCKER_DRIVER: overlay2
   GRADLE_OPTS: "-Dorg.gradle.daemon=false"
   TEST_REPORTS: "/builds/OpenLicht/eraser/eraser-base/build/reports/tests/test/"
+  TEST_LOG: "/builds/OpenLicht/eraser/eraser-base/logs/eraser-test.log"
   JACOCO_REPORT: "/builds/OpenLicht/eraser/eraser-base/build/reports/jacoco/test/jacocoTestReport.xml"
+  TESTCONTAINERS_RYUK_DISABLED: "true"
 
 before_script:
   - export GRADLE_USER_HOME=`pwd`/.gradle
 
 build:
-  image: gradle:jdk8
+  image: openjdk:8
+  tags:
+    - docker
   stage: build
   script:
     - ./gradlew --console=plain --build-cache assemble
@@ -24,7 +32,9 @@ build:
       - .gradle
 
 test:
-  image: gradle:jdk8
+  image: openjdk:8
+  tags:
+    - docker
   stage: test
   script:
     - ./gradlew --continue --console=plain check jacocoTestReport
@@ -37,11 +47,14 @@ test:
   artifacts:
     when: always
     paths:
+      - $TEST_LOG
       - $TEST_REPORTS
       - $JACOCO_REPORT
 
 coverage:
   image: python:3.7.1-alpine
+  tags:
+    - docker
   stage: report
   dependencies:
   - test

.gitmodules

+[submodule "ragdoc-view"]
+	path = ragdoc-view
+	url = ../../jastadd/ragdoc-view/

README.md

@@ -22,10 +22,7 @@ There are a few subprojects in this repository:
 
 ## Setup and Building
 
-This project uses Gradle as the build tool. Please use the provided Gradle wrappers instead of an installed one.
-
-- Clone the repository, either using [https](https://git-st.inf.tu-dresden.de/OpenLicht/eraser.git) or [ssh](git@git-st.inf.tu-dresden.de:OpenLicht/eraser.git)
-- (Optional) Build all subprojects: `./gradlew build`
+This project uses Gradle as the build tool. For detailed information, see [setup guidelines](/../wikis/setup)
 
 ## Running the demo
 

benchmark/.gitignore

+/build/
+logs/

benchmark/build.gradle

+apply plugin: 'application'
+
+dependencies {
+    compile project(':eraser-base')
+    compile project(':feedbackloop.learner_backup')
+    compile project(':datasets')
+    testImplementation 'junit:junit:4.12'
+    // https://mvnrepository.com/artifact/org.hamcrest/java-hamcrest
+    // https://mvnrepository.com/artifact/org.hamcrest/hamcrest-core
+    testCompile group: 'org.hamcrest', name: 'hamcrest-core', version: '1.3'
+    implementation group: 'com.opencsv', name: 'opencsv', version: '4.1'
+    // https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient
+    compile group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.0-alpha4'
+    testCompile group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.11.2'
+    // https://mvnrepository.com/artifact/org.apache.httpcomponents/fluent-hc
+    compile group: 'org.apache.httpcomponents', name: 'fluent-hc', version: '4.2.1'
+    // https://mvnrepository.com/artifact/junit/junit
+    testCompile group: 'junit', name: 'junit', version: '4.4'
+    
+}
+
+run {
+    mainClassName = 'de.tudresden.inf.st.eraser.benchmark.Main'
+    standardInput = System.in
+    if (project.hasProperty("appArgs")) {
+        args Eval.me(appArgs)
+    }
+}
+
+sourceSets {
+    main {
+        java {
+            srcDir 'src/main/java'
+        }
+    }
+}

benchmark/src/main/java/de/tudresden/inf/st/eraser/benchmark/Benchmark.java

+package de.tudresden.inf.st.eraser.benchmark;
+
+import com.opencsv.CSVReader;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.fluent.Request;
+import org.apache.http.entity.ContentType;
+import org.apache.http.util.EntityUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import java.io.File;
+import java.io.FileReader;
+import java.util.Arrays;
+
+public class Benchmark {
+
+    //read every 5 s from csv
+    //activity CSV
+    /**
+     * Col 1: smartphone acceleration x
+     * Col 2: smartphone acceleration y
+     * Col 3: smartphone acceleration z
+     * Col 4: smartphone rotation x
+     * Col 5: smartphone rotation y
+     * Col 6: smartphone rotation z
+     * Col 7: watch acceleration x
+     * Col 8: watch acceleration y
+     * Col 9: watch acceleration z
+     * Col 10: watch rotation x
+     * Col 11: watch rotation y
+     * Col 12: watch rotation z/*/
+    //preference CSV
+    /**
+     * Col 1: Activity
+     * Col 2: watch brightness range "bright, medium, dimmer, dark"*/
+
+    private String a_csv_file_path;
+    private String p_csv_file_path;
+    private static final Logger logger = LogManager.getLogger(Benchmark.class);
+    private static final String ERASER_ITEM_URI = "http://localhost:4567/model/items/";
+    //TODO ITEM_NAME HAS TO BE DISCUSSED
+    private static final String[] ACTIVITY_ITEM_NAME = {
+            "m_accel_x", "m_accel_y", "m_accel_z", "m_rotation_x",
+            "m_rotation_y", "m_rotation_z", "w_accel_x", "w_accel_y",
+            "w_accel_z", "w_rotation_x", "w_rotation_y", "w_rotation_z"};
+    private static final String[] PREFERENCE_ITEM_NAME = {
+           "w_brightness"
+    };
+    private static boolean flag2 = true;
+    //csv_type is activity or preference
+    Benchmark(String a_csv_file_path, String p_csv_file_path) {
+        this.a_csv_file_path = a_csv_file_path;
+        this.p_csv_file_path = p_csv_file_path;
+    }
+
+    void start(){
+        String PREFERENCE_URL = "http://localhost:4567/model/items/iris1_item/state";
+        String ACTIVITY_URL = "http://localhost:4567/activity/current";
+        int TIME_PERIOD = 5000;
+        File a_file;
+        File p_file;
+        FileReader a_file_reader;
+        FileReader p_file_reader;
+        CSVReader a_csv_reader;
+        CSVReader p_csv_reader;
+        String[] a_next_record;
+        String[] p_next_record;
+        boolean flag1;
+        a_file=new File(a_csv_file_path);
+        p_file= new File(p_csv_file_path);
+        try {
+            a_file_reader =new FileReader(a_file);
+            p_file_reader=new FileReader(p_file);
+            a_csv_reader = new CSVReader(a_file_reader);
+            p_csv_reader = new CSVReader(p_file_reader);
+            while ((((a_next_record = a_csv_reader.readNext())!= null) && flag2)){
+                try{Thread.sleep(TIME_PERIOD);}catch (InterruptedException e){e.printStackTrace();}
+                String[] values = Arrays.copyOf(a_next_record,12);
+                setNewValue(values, ACTIVITY_ITEM_NAME,"activity");
+                HttpResponse response= Request.Get(ACTIVITY_URL).execute().returnResponse();
+                String status = response.getStatusLine().toString();
+                if(status.contains("200")){
+                    flag1 = true;
+                    logger.info("activity should be (read direct from CSV): " + a_next_record[12]);
+                    logger.info(EntityUtils.toString(response.getEntity()));
+                    logger.info("get activity from web server: response 200 ok");
+                }else{
+                    flag1 = false;
+                    flag2 = false;
+                    logger.info("can not get the activity from the web server");
+                }
+                while((((p_next_record = p_csv_reader.readNext()) != null) && flag1)) {
+                    try{Thread.sleep(TIME_PERIOD);}catch (InterruptedException e){e.printStackTrace();}
+                    String[] values1 = Arrays.copyOf(p_next_record,2);
+                    setNewValue(values1, PREFERENCE_ITEM_NAME,"preference");
+                    HttpResponse response1= Request.Get(PREFERENCE_URL).execute().returnResponse();
+                    String status1=response1.getStatusLine().toString();
+                    if(status1.contains("200")){
+                        flag2 = true;
+                        logger.info("get the iris1_item preference from web server: response 200 ok, value is: "+EntityUtils.toString(response1.getEntity()));
+                    }else {flag2 = false;
+                    logger.info("can not get the iris1_item from the web server");}
+                    break;
+                }
+            }
+        }
+        catch (Exception e){
+            e.printStackTrace();
+        }
+    }
+    private void setNewValue(String[] values, String[] name, String file_typ){
+        if(file_typ.equals("activity"))
+        {
+            int i = 0;
+            for(String value : values){
+                String uri = ERASER_ITEM_URI + name[i] + "/state";
+                try {
+                    HttpResponse httpResponse = Request.Put(uri)
+                            .bodyString(value, ContentType.TEXT_PLAIN)
+                            .execute().returnResponse();
+                    String status=httpResponse.getStatusLine().toString();
+                    if(status.contains("200")){
+                        logger.info("put activity input name: "+name[i]+", value: "+value+"to web server: response 200 ok");
+                    }else{
+                        logger.info("can not put activity inputs to rest server");
+                    }
+                }catch (Exception e){
+                    e.printStackTrace();
+                }
+                i+=1;
+            }
+        }else{
+            String uri= ERASER_ITEM_URI + "w_brightness" +"/state";
+            try {
+                HttpResponse httpResponse = Request.Put(uri)
+                        .bodyString(values[1], ContentType.TEXT_PLAIN)
+                        .execute().returnResponse();
+                String put_response=httpResponse.getStatusLine().toString();
+                if (put_response.contains("200")){logger.info("put w_brightness to web server: response 200 ok");}else{
+                    logger.info("can not put w_brightness to rest server");
+                }
+            }catch (Exception e){
+                e.printStackTrace();
+            }
+        }
+    }
+}
\ No newline at end of file

benchmark/src/main/java/de/tudresden/inf/st/eraser/benchmark/Main.java

+package de.tudresden.inf.st.eraser.benchmark;
+
+public class Main {
+
+    public static void main(String[] args) {
+        String A_CSV_FILE_PATH = "../datasets/backup/activity_data.csv";
+        String P_CSV_FILE_PATH = "../datasets/backup/preference_data.csv";
+        Benchmark benchmark=new Benchmark(A_CSV_FILE_PATH,P_CSV_FILE_PATH);
+        benchmark.start();
+    }
+}
+

benchmark/src/main/resources/log4j2.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration>
+    <Appenders>
+        <Console name="Console">
+            <PatternLayout pattern="%highlight{%d{HH:mm:ss.SSS} %-5level} %c{1.} - %msg%n"/>
+        </Console>
+        <RollingFile name="RollingFile" fileName="logs/eraser.log"
+                    filePattern="logs/eraser-%i.log">
+            <PatternLayout pattern="%d{HH:mm:ss.SSS} %-5level %logger{36} - %msg%n"/>
+            <Policies>
+                <OnStartupTriggeringPolicy/>
+            </Policies>
+            <DefaultRolloverStrategy max="20"/>
+        </RollingFile>
+    </Appenders>
+    <Loggers>
+        <Root level="debug">
+            <AppenderRef ref="Console"/>
+            <AppenderRef ref="RollingFile"/>
+        </Root>
+    </Loggers>
+</Configuration>

build.gradle

@@ -4,7 +4,7 @@ plugins {
 
 allprojects  {
 	group = 'de.tudresden.inf.st'
-	version = '1.0.0-SNAPSHOT'
+	version = '0.1'
 }
 
 subprojects {
@@ -31,4 +31,16 @@ subprojects {
 	artifacts {
 		testArtifacts testJar
 	}
+
+	repositories {
+		mavenCentral()
+	}
+
+	dependencies {
+		compile group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.11.2'
+		compile group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.11.2'
+		testCompile group: 'junit', name: 'junit', version: '4.12'
+		testCompile group: 'org.hamcrest', name: 'hamcrest-junit', version: '2.0.0.0'
+	}
+
 }

cloc/.gitignore

@@ -2,3 +2,4 @@ eraser-base.*
 eraser-base-*
 feedbackloop.txt
 my_definitions.txt
+logs/

commons.color/.gitignore

 /build/
 /bin/
+logs/

commons.color/build.gradle

-repositories {
-    mavenCentral()
-}
-
-apply plugin: 'java'
-
-sourceCompatibility = 1.8
-
 dependencies {
-    compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.9.8'
-    compile group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.11.1'
-    compile group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.11.1'
-    compile 'org.apache.commons:commons-math3:3.6.1'
-    testCompile group: 'junit', name: 'junit', version: '4.12'
-    testCompile group: 'org.hamcrest', name: 'hamcrest-junit', version: '2.0.0.0'
+    compile group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1'
 }
 
 sourceSets {

eraser-base/.gitignore

@@ -3,3 +3,5 @@ src/gen/
 src/gen-res/
 .test-tmp
 /bin/
+logs/
+/doc/

eraser-base/build.gradle

-repositories {
-    mavenCentral()
-}
-
 apply plugin: 'jastadd'
 apply plugin: 'application'
 apply plugin: 'jacoco'
+apply plugin: 'idea'
+apply plugin: 'distribution'
 
 dependencies {
     compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.9.8'
     compile group: 'net.sf.beaver', name: 'beaver-rt', version: '0.9.11'
-    compile group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.11.1'
-    compile group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.11.1'
-    compile 'org.fusesource.mqtt-client:mqtt-client:1.14'
-    compile 'org.apache.commons:commons-math3:3.6.1'
-    compile 'org.influxdb:influxdb-java:2.14'
-    testCompile group: 'junit', name: 'junit', version: '4.12'
+    compile group: 'org.fusesource.mqtt-client', name: 'mqtt-client', version: '1.15'
+    compile group: 'org.influxdb', name: 'influxdb-java', version: '2.15'
+    testCompile group: 'org.testcontainers', name: 'testcontainers', version: '1.11.2'
+    testCompile group: 'org.testcontainers', name: 'influxdb', version: '1.11.2'
+    testCompile group: 'org.apache.logging.log4j', name: 'log4j-slf4j-impl', version: '2.11.2'
 }
 
 buildscript {
     repositories.mavenLocal()
     repositories.mavenCentral()
     dependencies {
-        classpath 'org.jastadd:jastaddgradle:1.13.2'
+        classpath group: 'org.jastadd', name: 'jastaddgradle', version: '1.13.2'
     }
 }
 
@@ -47,18 +44,22 @@ jacocoTestReport {
     }
 }
 
-task preprocess(type: JavaExec) {
-    group = 'Build'
-    main = "-jar"
-    args = [
-            "libs/relast-compiler.jar",
-            "./src/main/jastadd/main.relast",
+def relastFiles = fileTree('src/main/jastadd/') {
+    include '**/*.relast' }.toList().toArray()
+String[] relastArguments = [
+        "libs/relast.jar",
+        "--grammarName=./src/main/jastadd/mainGen",
+        "--useJastAddNames",
         "--listClass=RefList",
         "--jastAddList=JastAddList",
         "--file"
 ]
+task preprocess(type: JavaExec) {
+    group = 'Build'
+    main = "-jar"
+    args relastArguments + relastFiles
 
-    inputs.files file("./src/main/jastadd/main.relast")
+    inputs.files relastFiles
     outputs.files file("./src/main/jastadd/mainGen.ast"), file("./src/main/jastadd/mainGen.jadd")
 }
 
@@ -79,6 +80,8 @@ jastadd {
     parser.genDir = "src/gen/java/de/tudresden/inf/st/eraser/jastadd/parser"
 }
 
+idea.module.generatedSourceDirs += file('src/gen/java')
+
 sourceSets.main {
     java {
         srcDir 'src/gen/java'
@@ -90,32 +93,32 @@ javadoc {
     failOnError = false
 }
 
-task RagDoll(type: Javadoc, dependsOn:javadoc) {
-    doFirst {
-        options.addStringOption("ragroot", "./src/main/jastadd")
-    }
-    group = "documentation"
-    description = 'create a RagDoll documentation'
-    classpath = javadoc.classpath
-    destinationDir = javadoc.destinationDir
-    excludes = javadoc.excludes
-    executable = javadoc.executable
-    failOnError = false
-    includes = javadoc.includes
-    options.doclet = "ragdoll.RagDollDoclet"
-    options.docletpath = files('libs/RagDoll.jar').asList()
+String[] arguments = ["libs/rd-builder.jar", "-d", "doc/"]
+def allSrcFiles = sourceSets.main.allSource.findAll { it.name.endsWith('java') }.toArray()
+def ragdocViewSrcData = '../ragdoc-view/src/data/'
 
-    source = javadoc.source
-    options.linkSource = true
+task ragdoc(type: JavaExec, dependsOn: assemble, overwrite: true) {
+    group = 'documentation'
+    description = 'Create ragdoc json documentation files'
+    main = "-jar"
+    args arguments + allSrcFiles
+}
 
-    // title not working for some reason
-    title = ""
-    doLast {
-        println "Visit: file://" + destinationDir + "/index.html"
+task cleanRagdoc(type: Delete) {
+    group = 'documentation'
+    delete fileTree(ragdocViewSrcData + '/*')
 }
+
+task copyRagdoc(type: Copy, dependsOn: cleanRagdoc) {
+    group = 'documentation'
+    description = 'Copy ragdoc json documentation files to ragdoc-viewer'
+    from 'doc/'
+    into ragdocViewSrcData
+    eachFile { println it.file }
 }
 
 generateAst.dependsOn preprocess
+generateAst.inputs.files file("./src/main/jastadd/mainGen.ast"), file("./src/main/jastadd/mainGen.jadd")
 //compileJava.dependsOn jastadd
 //
 //// always run jastadd

eraser-base/src/main/jastadd/AdditionalTypes.jadd

 aspect AdditionalTypes {
   public class StringList extends beaver.Symbol implements Iterable<String> {
-    private java.util.List<String> delegatee = new java.util.ArrayList<String>();
+    private java.util.Deque<String> delegatee = new java.util.ArrayDeque<>();
 
     public java.util.Iterator<String> iterator() {
-      return delegatee.iterator();
+      return delegatee.descendingIterator();
     }
 
     public void add(String s) {
       delegatee.add(s);
     }
   }
+
+  public class TypedKeyMap<T> extends beaver.Symbol implements Iterable<AbstractMap.SimpleEntry<T, String>> {
+    private java.util.Deque<AbstractMap.SimpleEntry<T, String>> delegatee = new java.util.ArrayDeque<>();
+
+    public java.util.Iterator<AbstractMap.SimpleEntry<T, String>> iterator() {
+      return delegatee.descendingIterator();
+    }
+
+    public void put(T key, String value) {
+      delegatee.add(new AbstractMap.SimpleEntry<>(key, value));
+    }
+  }
+
+  public class StringKeyMap extends TypedKeyMap<String> { }
+
+  public class IntegerKeyMap extends TypedKeyMap<Integer> { }
 }

eraser-base/src/main/jastadd/DecisionTree.jrag

@@ -4,26 +4,28 @@ aspect DecisionTree {
   public class DecisionTreeLeaf implements Leaf { }
 
   //--- classify ---
-  syn Leaf DecisionTreeRoot.classify() {
+  syn DecisionTreeLeaf DecisionTreeRoot.classify() {
     return getRootRule().classify();
   }
 
-  syn Leaf DecisionTreeElement.classify();
+  syn DecisionTreeLeaf DecisionTreeElement.classify();
 
-  syn Leaf DecisionTreeRule.classify();
+  syn DecisionTreeLeaf DecisionTreeRule.classify();
 
-  syn Leaf ItemStateCheckRule.classify() {
+  syn DecisionTreeLeaf ItemStateCheckRule.classify() {
     boolean chooseLeft = getItemStateCheck().holds();
     return (chooseLeft ? getLeft() : getRight()).classify();
   }
 
-  syn Leaf DecisionTreeLeaf.classify() = this;
+  syn DecisionTreeLeaf DecisionTreeLeaf.classify() = this;
 
   //--- holds ---
-  syn boolean ItemStateCheck.holds();
+  syn boolean ItemStateCheck.holds() = holdsFor(getItem());
 
-  syn boolean ItemStateNumberCheck.holds() {
-    double actual = getItem().getStateAsDouble();
+  //--- holdsFor ---
+  syn boolean ItemStateCheck.holdsFor(Item item);
+  eq ItemStateNumberCheck.holdsFor(Item item) {
+    double actual = item.getStateAsDouble();
     double expected = getValue();
     boolean result;
     switch (getComparator()) {
@@ -38,12 +40,33 @@ aspect DecisionTree {
     return result;
   }
 
-  syn boolean ItemStateStringCheck.holds() {
+  eq ItemStateStringCheck.holdsFor(Item item) {
     // TODO implement this
     System.err.println("ItemStateStringCheck in Decision Tree not implemented!");
     return false;
   }
 
+  //--- computePreferences ---
+  syn List<ItemPreference> DecisionTreeLeaf.computePreferences() {
+    // iterate over preference of this leaf, and all its parents and ancestors
+    List<ItemPreference> result = new ArrayList<>();
+    Set<Item> seenItems = new HashSet<>();
+    List<DecisionTreeElement> ancestors = ancestors();
+    for (ItemPreference pref : getPreferenceList()) {
+      result.add(pref);
+      seenItems.add(pref.getItem());
+    }
+    for (DecisionTreeElement ancestor : ancestors) {
+      for (ItemPreference pref : ancestor.getPreferenceList()) {
+        if (!seenItems.contains(pref.getItem())) {
+          result.add(pref);
+          seenItems.add(pref.getItem());
+        }
+      }
+    }
+    return result;
+  }
+
   //--- ancestors ---
   inh List<DecisionTreeElement> DecisionTreeElement.ancestors();
   eq DecisionTreeRule.getLeft().ancestors() {
@@ -57,5 +80,11 @@ aspect DecisionTree {
     return result;
   }
   eq DecisionTreeRoot.getRootRule().ancestors() = new ArrayList();
+  eq DummyMachineLearningModel.getCurrent().ancestors() = Collections.emptyList();
+
+  public void DecisionTreeRoot.connectItems(List<String> itemNames) {
+    // TODO walk through the tree using depth-first-search
+    throw new UnsupportedOperationException();
+  }
 
 }