diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000000000000000000000000000000000000..71028dae684d7449a054c33c53adaf25035cf6b3
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 Georg Hinkel
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
index fd4621f455bd47c2abdc7e6870660105dcbc1147..7ad53aa94e6c9b3eedf21fb7b33a7a9e43b2e50a 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,51 @@
-# ttc2019-tt2bdd
-Free form TTC contest (Truth tables to Binary Decision Diagrams, from ATL Zoo)
+# The TTC 2018 Social Media Case
+
+## Case description
+
+The `docs/2018_TTC_Live.pdf` file contains the [case description](https://github.com/TransformationToolContest/ttc2018liveContest/raw/master/docs/2018_TTC_Live.pdf).
+
+## Prerequisites
+
+* 64-bit operating system
+* Python 3.3 or higher
+* R
+
+## Solution Prerequisites
+
+* NMF: You need to install [.NET Core 2.0](https://www.microsoft.com/net/download/linux-package-manager/ubuntu16-04/sdk-current)
+
+Add your prerequisites here!
+
+## Using the framework
+
+The `scripts` directory contains the `run.py` script.
+At a first glance, invoke it without any arguments so that the solution will be built, benchmarked, running times visualized and the results compared to the reference solution's.
+One might fine tune the script for the following purposes:
+* `run.py -b` -- builds the projects
+* `run.py -b -s` -- builds the projects without testing
+* `run.py -g` -- generates the instance models
+* `run.py -m` -- run the benchmark without building
+* `run.py -v` -- visualizes the results of the latest benchmark
+* `run.py -e` -- compare results to the reference output. The benchmark shall already been executed using `-m`.
+* `run.py -m -e` -- run benchmark without building, then extract and compare results to the reference output
+* `run.py -t` -- build the project and run tests (usually unit tests as defined for the given solution)
+
+The `config` directory contains the configuration for the scripts:
+* `config.json` -- configuration for the model generation and the benchmark
+ * *Note:* the timeout as set in the benchmark configuration (default: 6000 seconds) applies to the gross cumulative runtime of the tool for a given changeset and update sequences. This also includes e.g. Initialization time which is not required by the benchmark framework to be measured.
+ Timeout is only applied to the solutions' run phase (see `-m` for `run.py`), so it is not applied to e.g. the build phase (see `-b` for `run.py`).
+* `reporting.json` -- configuration for the visualization
+
+### Running the benchmark
+
+The script runs the benchmark for the given number of runs, for the specified tools and change sequences.
+
+The benchmark results are stored in a CSV file. The header for the CSV file is stored in the `output/header.csv` file.
+
+## Reporting and visualization
+
+Make sure you read the `README.md` file in the `reporting` directory and install all the requirements for R.
+
+## Implementing the benchmark for a new tool
+
+To implement a tool, you need to create a new directory in the solutions directory and give it a suitable name.
diff --git a/config/config.json b/config/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..a3933dd4f9532961176b9a21592c82c8d2376a0f
--- /dev/null
+++ b/config/config.json
@@ -0,0 +1,6 @@
+{
+ "Tools": ["ATL"],
+ "Sequences": 20,
+ "Runs": 5,
+ "Timeout": 6000
+}
diff --git a/config/reporting.json b/config/reporting.json
new file mode 100644
index 0000000000000000000000000000000000000000..cd3e7da23410f437dde08e48b4116497c8d41b26
--- /dev/null
+++ b/config/reporting.json
@@ -0,0 +1,21 @@
+{
+ "Dimensions": {
+ "X_Dimensions": {
+ "Size": true,
+ "Iteration": false
+ },
+ "Groups": {
+ "View": false,
+ "Tool": true
+ }
+ },
+ "Summarize_Functions": [
+ {
+ "Name": "Runtime",
+ "Phases": [
+ "Run"
+ ]
+ }
+ ],
+ "Extension": ["png", "pdf"]
+}
diff --git a/expected-results/README.md b/expected-results/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..fb3f7304612d6152192bc9eb424f81060c6b677b
--- /dev/null
+++ b/expected-results/README.md
@@ -0,0 +1 @@
+This folder contains the reference result data for the benchmark cases.
diff --git a/metamodels/Ecore.ecore b/metamodels/Ecore.ecore
new file mode 100644
index 0000000000000000000000000000000000000000..15a3569fbf2ed7be4764087725a9364b9899f9ee
--- /dev/null
+++ b/metamodels/Ecore.ecore
@@ -0,0 +1,526 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ecore:EPackage xmi:version="2.0" xmlns:xmi="http://www.omg.org/XMI" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:ecore="http://www.eclipse.org/emf/2002/Ecore" name="ecore" nsURI="http://www.eclipse.org/emf/2002/Ecore" nsPrefix="ecore">
+ <eClassifiers xsi:type="ecore:EClass" name="EAttribute" eSuperTypes="#//EStructuralFeature">
+ <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore">
+ <details key="constraints" value="ConsistentTransient"/>
+ </eAnnotations>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="iD" eType="#//EBoolean"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eAttributeType" lowerBound="1"
+ eType="#//EDataType" changeable="false" volatile="true" transient="true" derived="true"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EClass" name="EAnnotation" eSuperTypes="#//EModelElement">
+ <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore">
+ <details key="constraints" value="WellFormedSourceURI"/>
+ </eAnnotations>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="source" eType="#//EString"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="details" upperBound="-1"
+ eType="#//EStringToStringMapEntry" containment="true" resolveProxies="false"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eModelElement" eType="#//EModelElement"
+ transient="true" resolveProxies="false" eOpposite="#//EModelElement/eAnnotations"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="contents" upperBound="-1"
+ eType="#//EObject" containment="true" resolveProxies="false"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="references" upperBound="-1"
+ eType="#//EObject"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EClass" name="EClass" eSuperTypes="#//EClassifier">
+ <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore">
+ <details key="constraints" value="InterfaceIsAbstract AtMostOneID UniqueFeatureNames UniqueOperationSignatures NoCircularSuperTypes WellFormedMapEntryClass ConsistentSuperTypes DisjointFeatureAndOperationSignatures"/>
+ </eAnnotations>
+ <eOperations name="isSuperTypeOf" eType="#//EBoolean">
+ <eParameters name="someClass" eType="#//EClass"/>
+ </eOperations>
+ <eOperations name="getFeatureCount" eType="#//EInt"/>
+ <eOperations name="getEStructuralFeature" eType="#//EStructuralFeature">
+ <eParameters name="featureID" eType="#//EInt"/>
+ </eOperations>
+ <eOperations name="getFeatureID" eType="#//EInt">
+ <eParameters name="feature" eType="#//EStructuralFeature"/>
+ </eOperations>
+ <eOperations name="getEStructuralFeature" eType="#//EStructuralFeature">
+ <eParameters name="featureName" eType="#//EString"/>
+ </eOperations>
+ <eOperations name="getOperationCount" eType="#//EInt"/>
+ <eOperations name="getEOperation" eType="#//EOperation">
+ <eParameters name="operationID" eType="#//EInt"/>
+ </eOperations>
+ <eOperations name="getOperationID" eType="#//EInt">
+ <eParameters name="operation" eType="#//EOperation"/>
+ </eOperations>
+ <eOperations name="getOverride" eType="#//EOperation">
+ <eParameters name="operation" eType="#//EOperation"/>
+ </eOperations>
+ <eOperations name="getFeatureType" eType="#//EGenericType">
+ <eParameters name="feature" eType="#//EStructuralFeature"/>
+ </eOperations>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="abstract" eType="#//EBoolean"/>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="interface" eType="#//EBoolean"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eSuperTypes" upperBound="-1"
+ eType="#//EClass" unsettable="true">
+ <eAnnotations source="http://www.eclipse.org/emf/2002/GenModel">
+ <details key="suppressedIsSetVisibility" value="true"/>
+ <details key="suppressedUnsetVisibility" value="true"/>
+ </eAnnotations>
+ </eStructuralFeatures>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eOperations" upperBound="-1"
+ eType="#//EOperation" containment="true" resolveProxies="false" eOpposite="#//EOperation/eContainingClass"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eAllAttributes" upperBound="-1"
+ eType="#//EAttribute" changeable="false" volatile="true" transient="true"
+ derived="true"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eAllReferences" upperBound="-1"
+ eType="#//EReference" changeable="false" volatile="true" transient="true"
+ derived="true"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eReferences" upperBound="-1"
+ eType="#//EReference" changeable="false" volatile="true" transient="true"
+ derived="true"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eAttributes" upperBound="-1"
+ eType="#//EAttribute" changeable="false" volatile="true" transient="true"
+ derived="true"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eAllContainments" upperBound="-1"
+ eType="#//EReference" changeable="false" volatile="true" transient="true"
+ derived="true"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eAllOperations" upperBound="-1"
+ eType="#//EOperation" changeable="false" volatile="true" transient="true"
+ derived="true"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eAllStructuralFeatures"
+ upperBound="-1" eType="#//EStructuralFeature" changeable="false" volatile="true"
+ transient="true" derived="true"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eAllSuperTypes" upperBound="-1"
+ eType="#//EClass" changeable="false" volatile="true" transient="true" derived="true"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eIDAttribute" eType="#//EAttribute"
+ changeable="false" volatile="true" transient="true" derived="true" resolveProxies="false"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eStructuralFeatures" upperBound="-1"
+ eType="#//EStructuralFeature" containment="true" resolveProxies="false" eOpposite="#//EStructuralFeature/eContainingClass"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eGenericSuperTypes" upperBound="-1"
+ eType="#//EGenericType" unsettable="true" containment="true" resolveProxies="false">
+ <eAnnotations source="http://www.eclipse.org/emf/2002/GenModel">
+ <details key="suppressedIsSetVisibility" value="true"/>
+ <details key="suppressedUnsetVisibility" value="true"/>
+ </eAnnotations>
+ </eStructuralFeatures>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eAllGenericSuperTypes"
+ upperBound="-1" eType="#//EGenericType" changeable="false" volatile="true"
+ transient="true" derived="true"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EClass" name="EClassifier" abstract="true" eSuperTypes="#//ENamedElement">
+ <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore">
+ <details key="constraints" value="WellFormedInstanceTypeName UniqueTypeParameterNames"/>
+ </eAnnotations>
+ <eOperations name="isInstance" eType="#//EBoolean">
+ <eParameters name="object" eType="#//EJavaObject"/>
+ </eOperations>
+ <eOperations name="getClassifierID" eType="#//EInt"/>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="instanceClassName" eType="#//EString"
+ volatile="true" unsettable="true">
+ <eAnnotations source="http://www.eclipse.org/emf/2002/GenModel">
+ <details key="suppressedIsSetVisibility" value="true"/>
+ <details key="suppressedUnsetVisibility" value="true"/>
+ </eAnnotations>
+ </eStructuralFeatures>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="instanceClass" changeable="false"
+ volatile="true" transient="true" derived="true">
+ <eGenericType eClassifier="#//EJavaClass">
+ <eTypeArguments/>
+ </eGenericType>
+ </eStructuralFeatures>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="defaultValue" eType="#//EJavaObject"
+ changeable="false" volatile="true" transient="true" derived="true"/>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="instanceTypeName" eType="#//EString"
+ volatile="true" unsettable="true">
+ <eAnnotations source="http://www.eclipse.org/emf/2002/GenModel">
+ <details key="suppressedIsSetVisibility" value="true"/>
+ <details key="suppressedUnsetVisibility" value="true"/>
+ </eAnnotations>
+ </eStructuralFeatures>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="ePackage" eType="#//EPackage"
+ changeable="false" transient="true" eOpposite="#//EPackage/eClassifiers"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eTypeParameters" upperBound="-1"
+ eType="#//ETypeParameter" containment="true"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EClass" name="EDataType" eSuperTypes="#//EClassifier">
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="serializable" eType="#//EBoolean"
+ defaultValueLiteral="true"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EClass" name="EEnum" eSuperTypes="#//EDataType">
+ <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore">
+ <details key="constraints" value="UniqueEnumeratorNames UniqueEnumeratorLiterals"/>
+ </eAnnotations>
+ <eOperations name="getEEnumLiteral" eType="#//EEnumLiteral">
+ <eParameters name="name" eType="#//EString"/>
+ </eOperations>
+ <eOperations name="getEEnumLiteral" eType="#//EEnumLiteral">
+ <eParameters name="value" eType="#//EInt"/>
+ </eOperations>
+ <eOperations name="getEEnumLiteralByLiteral" eType="#//EEnumLiteral">
+ <eParameters name="literal" eType="#//EString"/>
+ </eOperations>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eLiterals" upperBound="-1"
+ eType="#//EEnumLiteral" containment="true" resolveProxies="false" eOpposite="#//EEnumLiteral/eEnum"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EClass" name="EEnumLiteral" eSuperTypes="#//ENamedElement">
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="value" eType="#//EInt"/>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="instance" eType="#//EEnumerator"
+ transient="true"/>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="literal" eType="#//EString"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eEnum" eType="#//EEnum"
+ changeable="false" transient="true" resolveProxies="false" eOpposite="#//EEnum/eLiterals"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EClass" name="EFactory" eSuperTypes="#//EModelElement">
+ <eOperations name="create" eType="#//EObject">
+ <eParameters name="eClass" eType="#//EClass"/>
+ </eOperations>
+ <eOperations name="createFromString" eType="#//EJavaObject">
+ <eParameters name="eDataType" eType="#//EDataType"/>
+ <eParameters name="literalValue" eType="#//EString"/>
+ </eOperations>
+ <eOperations name="convertToString" eType="#//EString">
+ <eParameters name="eDataType" eType="#//EDataType"/>
+ <eParameters name="instanceValue" eType="#//EJavaObject"/>
+ </eOperations>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="ePackage" lowerBound="1"
+ eType="#//EPackage" transient="true" resolveProxies="false" eOpposite="#//EPackage/eFactoryInstance"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EClass" name="EModelElement" abstract="true">
+ <eOperations name="getEAnnotation" eType="#//EAnnotation">
+ <eParameters name="source" eType="#//EString"/>
+ </eOperations>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eAnnotations" upperBound="-1"
+ eType="#//EAnnotation" containment="true" resolveProxies="false" eOpposite="#//EAnnotation/eModelElement"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EClass" name="ENamedElement" abstract="true" eSuperTypes="#//EModelElement">
+ <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore">
+ <details key="constraints" value="WellFormedName"/>
+ </eAnnotations>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="name" eType="#//EString"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EClass" name="EObject">
+ <eOperations name="eClass" eType="#//EClass"/>
+ <eOperations name="eIsProxy" eType="#//EBoolean"/>
+ <eOperations name="eResource" eType="#//EResource"/>
+ <eOperations name="eContainer" eType="#//EObject"/>
+ <eOperations name="eContainingFeature" eType="#//EStructuralFeature"/>
+ <eOperations name="eContainmentFeature" eType="#//EReference"/>
+ <eOperations name="eContents">
+ <eGenericType eClassifier="#//EEList">
+ <eTypeArguments eClassifier="#//EObject"/>
+ </eGenericType>
+ </eOperations>
+ <eOperations name="eAllContents">
+ <eGenericType eClassifier="#//ETreeIterator">
+ <eTypeArguments eClassifier="#//EObject"/>
+ </eGenericType>
+ </eOperations>
+ <eOperations name="eCrossReferences">
+ <eGenericType eClassifier="#//EEList">
+ <eTypeArguments eClassifier="#//EObject"/>
+ </eGenericType>
+ </eOperations>
+ <eOperations name="eGet" eType="#//EJavaObject">
+ <eParameters name="feature" eType="#//EStructuralFeature"/>
+ </eOperations>
+ <eOperations name="eGet" eType="#//EJavaObject">
+ <eParameters name="feature" eType="#//EStructuralFeature"/>
+ <eParameters name="resolve" eType="#//EBoolean"/>
+ </eOperations>
+ <eOperations name="eSet">
+ <eParameters name="feature" eType="#//EStructuralFeature"/>
+ <eParameters name="newValue" eType="#//EJavaObject"/>
+ </eOperations>
+ <eOperations name="eIsSet" eType="#//EBoolean">
+ <eParameters name="feature" eType="#//EStructuralFeature"/>
+ </eOperations>
+ <eOperations name="eUnset">
+ <eParameters name="feature" eType="#//EStructuralFeature"/>
+ </eOperations>
+ <eOperations name="eInvoke" eType="#//EJavaObject" eExceptions="#//EInvocationTargetException">
+ <eParameters name="operation" eType="#//EOperation"/>
+ <eParameters name="arguments">
+ <eGenericType eClassifier="#//EEList">
+ <eTypeArguments/>
+ </eGenericType>
+ </eParameters>
+ </eOperations>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EClass" name="EOperation" eSuperTypes="#//ETypedElement">
+ <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore">
+ <details key="constraints" value="UniqueParameterNames UniqueTypeParameterNames NoRepeatingVoid"/>
+ </eAnnotations>
+ <eOperations name="getOperationID" eType="#//EInt"/>
+ <eOperations name="isOverrideOf" eType="#//EBoolean">
+ <eParameters name="someOperation" eType="#//EOperation"/>
+ </eOperations>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eContainingClass" eType="#//EClass"
+ changeable="false" transient="true" resolveProxies="false" eOpposite="#//EClass/eOperations"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eTypeParameters" upperBound="-1"
+ eType="#//ETypeParameter" containment="true"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eParameters" upperBound="-1"
+ eType="#//EParameter" containment="true" resolveProxies="false" eOpposite="#//EParameter/eOperation"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eExceptions" upperBound="-1"
+ eType="#//EClassifier" unsettable="true">
+ <eAnnotations source="http://www.eclipse.org/emf/2002/GenModel">
+ <details key="suppressedIsSetVisibility" value="true"/>
+ <details key="suppressedUnsetVisibility" value="true"/>
+ </eAnnotations>
+ </eStructuralFeatures>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eGenericExceptions" upperBound="-1"
+ eType="#//EGenericType" unsettable="true" containment="true" resolveProxies="false">
+ <eAnnotations source="http://www.eclipse.org/emf/2002/GenModel">
+ <details key="suppressedIsSetVisibility" value="true"/>
+ <details key="suppressedUnsetVisibility" value="true"/>
+ </eAnnotations>
+ </eStructuralFeatures>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EClass" name="EPackage" eSuperTypes="#//ENamedElement">
+ <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore">
+ <details key="constraints" value="WellFormedNsURI WellFormedNsPrefix UniqueSubpackageNames UniqueClassifierNames UniqueNsURIs"/>
+ </eAnnotations>
+ <eOperations name="getEClassifier" eType="#//EClassifier">
+ <eParameters name="name" eType="#//EString"/>
+ </eOperations>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="nsURI" eType="#//EString"/>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="nsPrefix" eType="#//EString"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eFactoryInstance" lowerBound="1"
+ eType="#//EFactory" transient="true" resolveProxies="false" eOpposite="#//EFactory/ePackage"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eClassifiers" upperBound="-1"
+ eType="#//EClassifier" containment="true" eOpposite="#//EClassifier/ePackage"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eSubpackages" upperBound="-1"
+ eType="#//EPackage" containment="true" eOpposite="#//EPackage/eSuperPackage"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eSuperPackage" eType="#//EPackage"
+ changeable="false" transient="true" eOpposite="#//EPackage/eSubpackages"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EClass" name="EParameter" eSuperTypes="#//ETypedElement">
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eOperation" eType="#//EOperation"
+ changeable="false" transient="true" resolveProxies="false" eOpposite="#//EOperation/eParameters"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EClass" name="EReference" eSuperTypes="#//EStructuralFeature">
+ <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore">
+ <details key="constraints" value="ConsistentOpposite SingleContainer ConsistentKeys ConsistentUnique ConsistentContainer"/>
+ </eAnnotations>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="containment" eType="#//EBoolean"/>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="container" eType="#//EBoolean"
+ changeable="false" volatile="true" transient="true" derived="true"/>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="resolveProxies" eType="#//EBoolean"
+ defaultValueLiteral="true"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eOpposite" eType="#//EReference"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eReferenceType" lowerBound="1"
+ eType="#//EClass" changeable="false" volatile="true" transient="true" derived="true"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eKeys" upperBound="-1"
+ eType="#//EAttribute"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EClass" name="EStructuralFeature" abstract="true"
+ eSuperTypes="#//ETypedElement">
+ <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore">
+ <details key="constraints" value="ValidDefaultValueLiteral"/>
+ </eAnnotations>
+ <eOperations name="getFeatureID" eType="#//EInt"/>
+ <eOperations name="getContainerClass">
+ <eGenericType eClassifier="#//EJavaClass">
+ <eTypeArguments/>
+ </eGenericType>
+ </eOperations>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="changeable" eType="#//EBoolean"
+ defaultValueLiteral="true"/>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="volatile" eType="#//EBoolean"/>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="transient" eType="#//EBoolean"/>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="defaultValueLiteral" eType="#//EString"/>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="defaultValue" eType="#//EJavaObject"
+ changeable="false" volatile="true" transient="true" derived="true"/>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="unsettable" eType="#//EBoolean"/>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="derived" eType="#//EBoolean"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eContainingClass" eType="#//EClass"
+ changeable="false" transient="true" resolveProxies="false" eOpposite="#//EClass/eStructuralFeatures"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EClass" name="ETypedElement" abstract="true" eSuperTypes="#//ENamedElement">
+ <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore">
+ <details key="constraints" value="ValidLowerBound ValidUpperBound ConsistentBounds ValidType"/>
+ </eAnnotations>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="ordered" eType="#//EBoolean"
+ defaultValueLiteral="true"/>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="unique" eType="#//EBoolean"
+ defaultValueLiteral="true"/>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="lowerBound" eType="#//EInt"/>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="upperBound" eType="#//EInt"
+ defaultValueLiteral="1"/>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="many" eType="#//EBoolean"
+ changeable="false" volatile="true" transient="true" derived="true"/>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="required" eType="#//EBoolean"
+ changeable="false" volatile="true" transient="true" derived="true"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eType" eType="#//EClassifier"
+ volatile="true" unsettable="true">
+ <eAnnotations source="http://www.eclipse.org/emf/2002/GenModel">
+ <details key="suppressedIsSetVisibility" value="true"/>
+ <details key="suppressedUnsetVisibility" value="true"/>
+ </eAnnotations>
+ </eStructuralFeatures>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eGenericType" eType="#//EGenericType"
+ volatile="true" unsettable="true" containment="true" resolveProxies="false">
+ <eAnnotations source="http://www.eclipse.org/emf/2002/GenModel">
+ <details key="suppressedIsSetVisibility" value="true"/>
+ <details key="suppressedUnsetVisibility" value="true"/>
+ </eAnnotations>
+ </eStructuralFeatures>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EBigDecimal" instanceClassName="java.math.BigDecimal">
+ <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+ <details key="baseType" value="http://www.w3.org/2001/XMLSchema#decimal"/>
+ </eAnnotations>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EBigInteger" instanceClassName="java.math.BigInteger">
+ <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+ <details key="baseType" value="http://www.w3.org/2001/XMLSchema#integer"/>
+ </eAnnotations>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EBoolean" instanceClassName="boolean">
+ <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+ <details key="baseType" value="http://www.w3.org/2001/XMLSchema#boolean"/>
+ </eAnnotations>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EBooleanObject" instanceClassName="java.lang.Boolean">
+ <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+ <details key="baseType" value="EBoolean"/>
+ <details key="name" value="EBoolean:Object"/>
+ </eAnnotations>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EByte" instanceClassName="byte">
+ <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+ <details key="baseType" value="http://www.w3.org/2001/XMLSchema#byte"/>
+ </eAnnotations>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EByteArray" instanceClassName="byte[]">
+ <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+ <details key="baseType" value="http://www.w3.org/2001/XMLSchema#hexBinary"/>
+ </eAnnotations>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EByteObject" instanceClassName="java.lang.Byte">
+ <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+ <details key="baseType" value="EByte"/>
+ <details key="name" value="EByte:Object"/>
+ </eAnnotations>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EChar" instanceClassName="char"/>
+ <eClassifiers xsi:type="ecore:EDataType" name="ECharacterObject" instanceClassName="java.lang.Character">
+ <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+ <details key="baseType" value="EChar"/>
+ <details key="name" value="EChar:Object"/>
+ </eAnnotations>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EDate" instanceClassName="java.util.Date"/>
+ <eClassifiers xsi:type="ecore:EDataType" name="EDiagnosticChain" instanceClassName="org.eclipse.emf.common.util.DiagnosticChain"
+ serializable="false"/>
+ <eClassifiers xsi:type="ecore:EDataType" name="EDouble" instanceClassName="double">
+ <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+ <details key="baseType" value="http://www.w3.org/2001/XMLSchema#double"/>
+ </eAnnotations>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EDoubleObject" instanceClassName="java.lang.Double">
+ <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+ <details key="baseType" value="EDouble"/>
+ <details key="name" value="EDouble:Object"/>
+ </eAnnotations>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EEList" instanceClassName="org.eclipse.emf.common.util.EList"
+ serializable="false">
+ <eTypeParameters name="E"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EEnumerator" instanceClassName="org.eclipse.emf.common.util.Enumerator"
+ serializable="false"/>
+ <eClassifiers xsi:type="ecore:EDataType" name="EFeatureMap" instanceClassName="org.eclipse.emf.ecore.util.FeatureMap"
+ serializable="false"/>
+ <eClassifiers xsi:type="ecore:EDataType" name="EFeatureMapEntry" instanceClassName="org.eclipse.emf.ecore.util.FeatureMap$Entry"
+ serializable="false"/>
+ <eClassifiers xsi:type="ecore:EDataType" name="EFloat" instanceClassName="float">
+ <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+ <details key="baseType" value="http://www.w3.org/2001/XMLSchema#float"/>
+ </eAnnotations>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EFloatObject" instanceClassName="java.lang.Float">
+ <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+ <details key="baseType" value="EFloat"/>
+ <details key="name" value="EFloat:Object"/>
+ </eAnnotations>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EInt" instanceClassName="int">
+ <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+ <details key="baseType" value="http://www.w3.org/2001/XMLSchema#int"/>
+ </eAnnotations>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EIntegerObject" instanceClassName="java.lang.Integer">
+ <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+ <details key="baseType" value="EInt"/>
+ <details key="name" value="EInt:Object"/>
+ </eAnnotations>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EJavaClass" instanceClassName="java.lang.Class">
+ <eTypeParameters name="T"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EJavaObject" instanceClassName="java.lang.Object"/>
+ <eClassifiers xsi:type="ecore:EDataType" name="ELong" instanceClassName="long">
+ <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+ <details key="baseType" value="http://www.w3.org/2001/XMLSchema#long"/>
+ </eAnnotations>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="ELongObject" instanceClassName="java.lang.Long">
+ <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+ <details key="baseType" value="ELong"/>
+ <details key="name" value="ELong:Object"/>
+ </eAnnotations>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EMap" instanceClassName="java.util.Map"
+ serializable="false">
+ <eTypeParameters name="K"/>
+ <eTypeParameters name="V"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EResource" instanceClassName="org.eclipse.emf.ecore.resource.Resource"
+ serializable="false"/>
+ <eClassifiers xsi:type="ecore:EDataType" name="EResourceSet" instanceClassName="org.eclipse.emf.ecore.resource.ResourceSet"
+ serializable="false"/>
+ <eClassifiers xsi:type="ecore:EDataType" name="EShort" instanceClassName="short">
+ <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+ <details key="baseType" value="http://www.w3.org/2001/XMLSchema#short"/>
+ </eAnnotations>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EShortObject" instanceClassName="java.lang.Short">
+ <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+ <details key="baseType" value="EShort"/>
+ <details key="name" value="EShort:Object"/>
+ </eAnnotations>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EString" instanceClassName="java.lang.String">
+ <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData">
+ <details key="baseType" value="http://www.w3.org/2001/XMLSchema#string"/>
+ </eAnnotations>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EClass" name="EStringToStringMapEntry" instanceClassName="java.util.Map$Entry">
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="key" eType="#//EString"/>
+ <eStructuralFeatures xsi:type="ecore:EAttribute" name="value" eType="#//EString"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="ETreeIterator" instanceClassName="org.eclipse.emf.common.util.TreeIterator"
+ serializable="false">
+ <eTypeParameters name="E"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EClass" name="EGenericType">
+ <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore">
+ <details key="constraints" value="ConsistentType ConsistentBounds ConsistentArguments"/>
+ </eAnnotations>
+ <eOperations name="isInstance" eType="#//EBoolean">
+ <eParameters name="object" eType="#//EJavaObject"/>
+ </eOperations>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eUpperBound" eType="#//EGenericType"
+ containment="true" resolveProxies="false"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eTypeArguments" upperBound="-1"
+ eType="#//EGenericType" containment="true" resolveProxies="false"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eRawType" lowerBound="1"
+ eType="#//EClassifier" changeable="false" transient="true" derived="true"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eLowerBound" eType="#//EGenericType"
+ containment="true" resolveProxies="false"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eTypeParameter" eType="#//ETypeParameter"
+ resolveProxies="false"/>
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eClassifier" eType="#//EClassifier"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EClass" name="ETypeParameter" eSuperTypes="#//ENamedElement">
+ <eStructuralFeatures xsi:type="ecore:EReference" name="eBounds" upperBound="-1"
+ eType="#//EGenericType" containment="true" resolveProxies="false"/>
+ </eClassifiers>
+ <eClassifiers xsi:type="ecore:EDataType" name="EInvocationTargetException" instanceClassName="java.lang.reflect.InvocationTargetException"
+ serializable="false"/>
+</ecore:EPackage>
diff --git a/models/.gitignore b/models/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..91711cb0f57d823ee9a3b65682b4b01a706b10f2
--- /dev/null
+++ b/models/.gitignore
@@ -0,0 +1 @@
+/1024/
diff --git a/output/header.csv b/output/header.csv
new file mode 100644
index 0000000000000000000000000000000000000000..3db9cb29150270a7ae23a5e246199df7a541efd1
--- /dev/null
+++ b/output/header.csv
@@ -0,0 +1 @@
+Tool;Size;RunIndex;Iteration;PhaseName;MetricName;MetricValue
diff --git a/reporting/README.md b/reporting/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..557cc2dd88a25c3fbc88c21f817f1f5bd6fcc00b
--- /dev/null
+++ b/reporting/README.md
@@ -0,0 +1,70 @@
+# TTC 2017 Smart Grids Benchmark reporting tools
+
+## Dependencies
+
+1. Install R: the `r-base`, `r-base-dev` packages are required. The code is tested with R 3.0 (you can check your version with the `R --version` command).
+
+ ```bash
+ sudo apt-get install -y r-base r-base-dev
+ sudo apt-get install -y libssl-dev
+ ```
+
+1. Install the required R packages by issuing the following command:
+
+ ```bash
+ cd reporting/
+ sudo R -f install.R
+ ```
+
+ As a consequence, the `ggplot2`, `jsonlite`, `plyr` packages will be installed.
+
+## Usage
+
+1. Adjust the configuration parameters in the `config/reporting.json` file.
+1. Use the `scripts/run.py -v` command.
+1. Alternatively, execute the following command from the `reporting` directory:
+
+ `Rscript generate_diagrams.R`
+
+### Configuration
+
+* An example of the configuration file can be seen below:
+
+ ```json
+{
+ "Dimensions": {
+ "X_Dimensions": {
+ "Size": true,
+ "Iteration": true
+ },
+ "Groups": {
+ "Query": true,
+ "Tool": true
+ }
+ },
+ "Summarize_Functions": [
+ {
+ "Phases": [
+ "check",
+ "transformation"
+ ],
+ "Metric": "time",
+ "Y_Label": "Time (ms)",
+ "Y_Axis_Scale": -6
+ },
+ {
+ "Phases": [
+ "result"
+ ],
+ "Metric": "scalar",
+ "Y_Label": "Invalid elements",
+ "Y_Axis_Scale": 0
+ }
+ ],
+ "Extension": ["png", "pdf"]
+}
+ ```
+
+* An arbitrary number of functions can be declared.
+
+* The dimension and group parameters can be changed independently from each other, which means for example every variable can be `true` at the same time. As a consequence, more diagrams will be generated.
diff --git a/reporting/check_results.R b/reporting/check_results.R
new file mode 100644
index 0000000000000000000000000000000000000000..e13d43260247e6ae5b383b24bb72a3d3f6097f18
--- /dev/null
+++ b/reporting/check_results.R
@@ -0,0 +1,25 @@
+expected = read.csv2("../expected-results/results.csv")
+actual = subset(read.csv2("../output/output.csv"), MetricName=="Elements")
+
+tools = unique(actual$Tool)
+
+for (tool in tools) {
+ tool_data = subset(actual, Tool==tool)
+ queries = unique(tool_data$View)
+ for (query in queries) {
+ query_data = subset(tool_data, View==query)
+ query_n = length(row.names(query_data))
+ for (i in 1:query_n) {
+ query.row = query_data[i,]
+ expected.row = subset(expected, ChangeSet==query.row$ChangeSet & View==query & Iteration==query.row$Iteration)
+
+ if (length(as.character(expected.row$MetricValue)) > 0) {
+ if (as.character(query.row$MetricValue) != as.character(expected.row$MetricValue)) {
+ print(paste(tool, "is wrong. Was ", query.row$MetricValue, "but expected", expected.row$MetricValue, "for change set", query.row$ChangeSet, "query", query, "iteration", query.row$Iteration))
+ }
+ } else {
+ print(paste("Warning:", tool, "produced the result", query.row$MetricValue, "but expected result is unavailable for change set", query.row$ChangeSet, "query", query, "iteration", query.row$Iteration))
+ }
+ }
+ }
+}
diff --git a/reporting/constants.R b/reporting/constants.R
new file mode 100644
index 0000000000000000000000000000000000000000..481367ff8dcdf82b73b610951ee646ade8c86d82
--- /dev/null
+++ b/reporting/constants.R
@@ -0,0 +1,30 @@
+resultsPath <- "../output/output.csv"
+
+# y-axis scaling
+scaleDivisor <- 10**6
+
+# output folder
+rootPath <- c("../diagrams/")
+
+# plot settings
+pointSize = 3
+lineSize = 1
+yLabel <- "Time (ms)"
+yAxis <- "Log10"
+
+labels <- list("1" = "1\n ",
+ "2" = "2\n ",
+ "4" = "4\n ",
+ "8" = "8\n ",
+ "16"= "16\n ",
+ "32"= "32\n ",
+ "64" = "64\n ",
+ "128" = "128\n ",
+ "256" = "256\n ",
+ "512" = "512\n ",
+ "1024" = "1024\n ",
+ "2048" = "2048\n ",
+ "4096" = "4096\n ",
+ "8192" = "8192\n ",
+ "16384" = "16384\n "
+ )
diff --git a/reporting/extract_results.R b/reporting/extract_results.R
new file mode 100644
index 0000000000000000000000000000000000000000..b3b0eeadf4b7f285f593327679daa24428e67438
--- /dev/null
+++ b/reporting/extract_results.R
@@ -0,0 +1,35 @@
+library("plyr", quietly=T, verbose=F, warn.conflicts=FALSE)
+source("constants.R")
+results <-read.csv2(resultsPath, header=TRUE)
+
+changes <- unique(results$ChangeSet)
+metricName <- "elements"
+
+for (change in changes){
+ subData <- subset(results, ChangeSet == change)
+ queries <- unique(subData$Query)
+ for (query in queries){
+ fileName <- paste("../results/",change, "-", query, ".csv", sep="")
+ if (file.exists(fileName)){
+ file.remove(fileName)
+ }
+
+ firstRow <- list("Size")
+ subData2 <- subset(subData, Query == query)
+ iterations <- unique(subData2$Iteration)
+ for (iter in iterations){
+ firstRow <- paste(firstRow, iter, sep="\t")
+ }
+ write(firstRow, fileName, append=TRUE)
+ sizes <- unique(subData2$Size)
+ for (size in sizes){
+ row <- list(size)
+ for (iter in iterations){
+ metricValue <- subset(subData2, Size==size & Iteration==iter & MetricName==metricName & RunIndex==1)$MetricValue
+ row <- paste(row, metricValue, sep="\t")
+ }
+ write(row, fileName, append=TRUE)
+ }
+
+ }
+}
diff --git a/reporting/functions.R b/reporting/functions.R
new file mode 100644
index 0000000000000000000000000000000000000000..2c1663533445862d6413e1e2c1d16afdf506b788
--- /dev/null
+++ b/reporting/functions.R
@@ -0,0 +1,125 @@
+source("theme.R")
+
+savePlot <-function(results, settings, phases, fileName){
+ phases <- unlist(phases)
+ first <- TRUE
+ for(phase in phases){
+
+ if (first == TRUE){
+ merged <- subset(results, PhaseName == phase) # selection on phase
+ first <- FALSE
+ }
+ else{
+ merged <- rbind(merged, subset(results, PhaseName == phase)) # merge back
+ }
+ }
+
+ if (nrow(merged) == 0) {
+ return()
+ }
+ if (settings@xDimension == "ChangeSet"){
+ # summarise the iterations
+ data <- ddply(merged, c("ChangeSet", "View", "Tool", "RunIndex", "MetricName"),
+ summarize, MetricValue=sum(MetricValue))
+ data <- ddply(data, c("ChangeSet", "View", "Tool", "MetricName"),
+ summarize, MetricValue=mean(MetricValue))
+ }
+ else if(settings@xDimension == "Iteration"){
+ data <- ddply(merged, c("View", "Tool", "ChangeSet", "RunIndex", "MetricName", "Iteration"),
+ summarize, MetricValue=sum(MetricValue))
+ data <- ddply(data, c("View", "Tool", "ChangeSet", "MetricName", "Iteration"),
+ summarize, MetricValue=median(MetricValue))
+ }
+ else {
+ return()
+ }
+ artifacts <- unique(data[[settings@xDimension]])
+ minValue <- min(data$MetricValue)
+ maxValue <- max(data$MetricValue)
+
+ xLabels <- c(artifacts)
+ plot <- ggplot(data,aes_string(x = settings@xDimension, y = settings@yDimension)) +
+ geom_line(aes_string(group = settings@group, colour=settings@group), size=lineSize) +
+ geom_point(aes_string(shape = settings@group, colour=settings@group), size=pointSize) +
+ scale_shape_manual(values=1:nlevels(data[[settings@group]])) +
+ ylab(settings@yLabel) +
+ ggtitle(label = settings@title) +
+ bwTheme
+
+ if (settings@xAxis == "Continuous"){
+ plot <- plot + scale_x_continuous(breaks = c(artifacts))
+ }
+ else if (settings@xAxis == "Log2"){
+ plot <- plot + scale_x_log10(breaks = c(artifacts), labels = xLabels)
+ }
+ if (minValue == 0){
+ print("The minimum metricvalue equals with 0. The plot cannot be generated.")
+ return()
+ }
+ if (settings@yAxis == "Continuous"){
+ plot <- plot + scale_y_continuous(breaks = seq(minValue, maxValue, by=round(maxValue/5,0)),
+ labels = seq(minValue, maxValue, by=round(maxValue/5,0)))
+ }
+ else if (settings@yAxis == "Log10"){
+ plot <- plot + scale_y_log10(breaks = 10^seq(round(log10(minValue)), round(log10(maxValue)), by=1),
+ labels = 10^seq(round(log10(minValue)), round(log10(maxValue)), by=1))
+ }
+ ggsave(plot,filename = fileName, width=10, height=5, dpi=192)
+ print(fileName)
+
+
+}
+
+validPhase <- function(results, functions){
+ uniquePhases <- unique(results$PhaseName)
+ for(func in functions){
+ for(phase in func){
+ if(phase %in% uniquePhases == FALSE){
+ return(FALSE)
+ }
+ }
+ }
+ return(TRUE)
+}
+
+validMetric <- function(results, metric){
+ if (metric %in% levels(results$MetricName)){
+ return(TRUE)
+ }
+ return(FALSE)
+}
+
+createFolders <- function(rootPath, subFolders){
+ if (file.exists(rootPath) == FALSE){
+ dir.create(rootPath)
+ }
+ for (folder in subFolders){
+ path <- paste(rootPath, folder, sep='')
+ if (file.exists(path) == FALSE){
+ dir.create(path)
+ }
+ }
+}
+
+concatPhases <- function(phases){
+ phases <- unlist(phases)
+ first <- TRUE
+ for(p in phases){
+ if(first){
+ merged <- p
+ first <- FALSE
+ }
+ else{
+ merged <- paste(merged, p, sep='+')
+ }
+ }
+ return(merged)
+}
+
+getXLabels <- function(artifacts){
+ ticks <- c()
+ for(size in artifacts){
+ ticks <- c(ticks, labels[[as.character(size)]])
+ }
+ return(ticks)
+}
diff --git a/reporting/install.R b/reporting/install.R
new file mode 100644
index 0000000000000000000000000000000000000000..5fc125685bb7385360101d776aa03928cd8342bc
--- /dev/null
+++ b/reporting/install.R
@@ -0,0 +1,5 @@
+install.packages("ggplot2", repos="http://cran.rstudio.com/");
+install.packages("jsonlite", repos="http://cran.rstudio.com/");
+install.packages("devtools", repos="http://cran.rstudio.com/")
+require(devtools)
+install_version("plyr", version = "1.8.1", repos = "http://cran.rstudio.com/")
diff --git a/reporting/memory.r b/reporting/memory.r
new file mode 100644
index 0000000000000000000000000000000000000000..a4a8c905777c92d63588a18858245f222408ccde
--- /dev/null
+++ b/reporting/memory.r
@@ -0,0 +1,42 @@
+library("ggplot2")
+library("plyr")
+
+results <- read.csv2("../output/output.csv", row.names = NULL)
+
+testset = ddply(results, c("Tool", "RunIndex", "ChangeSet", "View"), MetricValue=max(MetricValue))
+testset = ddply(testset, c("Tool", "ChangeSet", "View"), summarize, Memory=mean(MetricValue)/(1024*1024))
+
+pointSize = 3
+lineSize = 1
+
+bwTheme <- theme(text=element_text(family="Helvetica", size=16),
+ panel.background = element_rect(fill="#FFFFFF"),
+ legend.position="bottom",
+ legend.direction="horizontal",
+ panel.grid.major = element_line(size=0.3, colour="#333333"),
+ panel.grid.minor = element_line(size=0.15, colour="#CCCCCC"),
+ axis.text.x = element_text(colour="black"),
+ axis.text.y = element_text(colour="black"),
+ legend.position="right"
+)
+
+for (query in unique(testset$View)) {
+
+ data = subset(testset, View==query)
+
+ sizes <- unique(data$ChangeSet)
+ minValue <- min(data$Memory)
+ maxValue <- max(data$Memory)
+
+plot <- ggplot(data, aes_string(x="ChangeSet",y="Memory"))
+plot <- plot + geom_line(aes_string(group="Tool",colour="Tool"),size=lineSize)
+plot <- plot + geom_point(aes_string(shape="Tool", colour="Tool"), size=pointSize)
+plot <- plot + scale_shape_manual(values=1:4) + ylab("Memory (Mbyte)")
+plot <- plot + xlab("ChangeSet")
+plot <- plot + bwTheme
+plot <- plot + scale_y_log10(breaks = 10^seq(round(log10(minValue)), round(log10(maxValue)), by=1),
+ labels = 10^seq(round(log10(minValue)), round(log10(maxValue)), by=1))
+plot <- plot + ggtitle(paste(query, ", working set", sep=""))
+
+ggsave(plot, filename=paste(query, "-memory.pdf", sep=""), width=7, height=4, dpi=192)
+}
\ No newline at end of file
diff --git a/reporting/plot.R b/reporting/plot.R
new file mode 100644
index 0000000000000000000000000000000000000000..ae36a205abcc8c6942e4fa69f06993cb4b89e665
--- /dev/null
+++ b/reporting/plot.R
@@ -0,0 +1,93 @@
+PlotSettings <- setClass(
+ "PlotSettings",
+
+ slots = c(
+ xDimension = "character",
+ yDimension = "character",
+ group = "character",
+ xLabel = "character",
+ yLabel = "character",
+ title = "character",
+ xAxis = "character",
+ yAxis = "character"
+ ),
+
+ prototype=list(
+ xDimension = "",
+ yDimension = "",
+ group = "",
+ xLabel = "",
+ yLabel = "",
+ title = "",
+ xAxis = "",
+ yAxis = ""
+ )
+ )
+
+setGeneric(name="setDimensions",
+ def=function(theObject, xDimension, yDimension){
+ standardGeneric("setDimensions")
+ }
+)
+
+setMethod(f="setDimensions",
+ signature="PlotSettings",
+ definition=function(theObject, xDimension, yDimension){
+ theObject@xDimension <- xDimension
+ theObject@yDimension <- yDimension
+ return(theObject)
+ })
+
+setGeneric(name="setGroup",
+ def=function(theObject, group){
+ standardGeneric("setGroup")
+ }
+)
+
+setMethod(f="setGroup",
+ signature="PlotSettings",
+ definition=function(theObject, group){
+ theObject@group <- group
+ return(theObject)
+ })
+
+setGeneric(name="setTitle",
+ def=function(theObject, title){
+ standardGeneric("setTitle")
+ }
+)
+
+setMethod(f="setTitle",
+ signature="PlotSettings",
+ definition=function(theObject, title){
+ theObject@title <- title
+ return(theObject)
+ })
+
+setGeneric(name="setLabels",
+ def=function(theObject, xLabel, yLabel){
+ standardGeneric("setLabels")
+ }
+ )
+
+setMethod(f="setLabels",
+ signature="PlotSettings",
+ definition=function(theObject, xLabel, yLabel){
+ theObject@xLabel <- xLabel
+ theObject@yLabel <- yLabel
+ return(theObject)
+ })
+
+setGeneric(name="setAxis",
+ def=function(theObject, xAxis, yAxis){
+ standardGeneric("setAxis")
+ }
+)
+
+setMethod(f="setAxis",
+ signature="PlotSettings",
+ definition=function(theObject, xAxis, yAxis){
+ theObject@xAxis <- xAxis
+ theObject@yAxis <- yAxis
+ return(theObject)
+ })
diff --git a/reporting/theme.R b/reporting/theme.R
new file mode 100644
index 0000000000000000000000000000000000000000..fdafd77f3caf2801718523d59cd35483d7523372
--- /dev/null
+++ b/reporting/theme.R
@@ -0,0 +1,9 @@
+bwTheme <- theme(text=element_text(family="Helvetica", size=16),
+ panel.background = element_rect(fill="#FFFFFF"),
+ legend.position="bottom",
+ legend.direction="horizontal",
+ panel.grid.major = element_line(size=0.3, colour="#333333"),
+ panel.grid.minor = element_line(size=0.15, colour="#CCCCCC"),
+ axis.text.x = element_text(colour="black"),
+ axis.text.y = element_text(colour="black")
+)
\ No newline at end of file
diff --git a/reporting/visualize.R b/reporting/visualize.R
new file mode 100644
index 0000000000000000000000000000000000000000..50d2b94ded3976de5833f04b52ee635196763dc0
--- /dev/null
+++ b/reporting/visualize.R
@@ -0,0 +1,116 @@
+library("jsonlite", quietly=T, verbose=F, warn.conflicts=FALSE)
+library("ggplot2",quietly=T, verbose=F, warn.conflicts=FALSE)
+library("plyr", quietly=T, verbose=F, warn.conflicts=FALSE)
+source("functions.R")
+source("plot.R")
+source("constants.R")
+
+args <- commandArgs(trailingOnly = TRUE)
+configPath <- args[1]
+
+results <-read.csv2(resultsPath, header=TRUE, row.names = NULL)
+results$ChangeSet = as.factor(results$ChangeSet)
+
+config <- fromJSON(configPath)
+
+if (validPhase(results, config$Summarize_Functions$Phases) == FALSE){
+ print("Non existing phasename provided!")
+ quit()
+}
+
+index <- 0
+settings <- PlotSettings()
+uniqueScenarios <- unique(results$ChangeSet)
+
+for(row in 1:nrow(config$Summarize_Functions)){
+ phases <- config$Summarize_Functions[row,]$Phases
+ name <- config$Summarize_Functions[row,]$Name
+ index <- index + 1
+ for(scenario in uniqueScenarios){
+ metric <- "Time"
+ subData1 <- subset(results, ChangeSet==scenario & MetricName == metric)
+ subData1$MetricValue <- as.numeric(as.character(subData1$MetricValue)) * (10**-6)
+
+ if (config$Dimensions$Groups$View){
+ uniqueTools <- unique(subData1$Tool)
+ settings <- setGroup(settings, "View")
+ for(tool in uniqueTools){
+ subData2 <- subset(subData1, Tool==tool)
+
+ if (config$Dimensions$X_Dimensions$ChangeSet){
+ title <- paste(tool, ", ", scenario, ", Function: ", concatPhases(phases), sep='')
+
+ settings <- setTitle(settings, title)
+ settings <- setDimensions(settings, "ChangeSet", "MetricValue")
+ settings <- setLabels(settings, "ChangeSet", "Time (ms)")
+ settings <- setAxis(settings, "Factor", yAxis)
+ for (extension in config$Extension){
+ fileName <- paste(rootPath, scenario, "-", tool, "-GroupBy-View-", metric, "-", name, ".", extension, sep='')
+ savePlot(subData2, settings, phases, fileName)
+ }
+ write.csv(subData2, file = paste(rootPath, scenario, "-", tool, "-GroupBy-View-", metric, "-", name, ".csv", sep=''))
+ }
+
+ if (config$Dimensions$X_Dimensions$Iteration){
+ uniqueChangeSets <-unique(subData2$ChangeSet)
+ for(changeSet in uniqueChangeSets){
+ subData3 <- subset(subData2, ChangeSet==changeSet)
+ title <- paste(tool, ", ChangeSet: ", changeSet, ", Function: ", concatPhases(phases), sep='')
+
+ settings <- setTitle(settings, title)
+ settings <- setDimensions(settings, "Iteration", "MetricValue")
+ settings <- setLabels(settings, "Iterations", "Time (ms)")
+ settings <- setAxis(settings, "Continuous", yAxis)
+ for (extension in config$Extension){
+ fileName <- paste(rootPath, scenario, "-", tool, "-changeSet-", changeSet, "-GroupBy-View-", metric, "-", name, ".", extension, sep='')
+ savePlot(subData3, settings, phases, fileName)
+ }
+ write.csv(subData3, file = paste(rootPath, scenario, "-", tool, "-ChangeSet-", changeSet, "-GroupBy-View-", metric, "-", name, ".csv", sep=''))
+ }
+ }
+ }
+ }
+ }
+
+ if (config$Dimensions$Groups$Tool){
+ metric <- "Time"
+ subData1 <- subset(results, MetricName == metric)
+ subData1$MetricValue <- as.numeric(as.character(subData1$MetricValue)) * (10**-6)
+
+ uniqueViews <- unique(subData1$View)
+ settings <- setGroup(settings, "Tool")
+ for(view in uniqueViews){
+ subData2 <- subset(subData1, View==view)
+
+ if (config$Dimensions$X_Dimensions$ChangeSet){
+ title <- paste(view, ", Function: ", concatPhases(phases), sep='')
+ settings <- setTitle(settings, title)
+ settings <- setDimensions(settings, "ChangeSet", "MetricValue")
+ settings <- setLabels(settings, "ChangeSet", "Time (ms)")
+ settings <- setAxis(settings, "Discrete", yAxis)
+ for (extension in config$Extension){
+ fileName <- paste(rootPath, view, "-GroupBy-Tool-",metric, "-", name, ".", extension, sep='')
+ savePlot(subData2, settings, phases, fileName)
+ }
+ write.csv(ddply(subData2, c("Tool", "ChangeSet"), summarise, N=length(MetricValue), mean=mean(MetricValue), sd=sd(MetricValue)), file = paste(rootPath, view, "-GroupBy-Tool-",metric, "-", name, ".csv", sep=''))
+ }
+
+ if (config$Dimensions$X_Dimensions$Iteration){
+ uniqueSizes <-unique(subData2$ChangeSet)
+ settings <- setDimensions(settings, "Iteration", "MetricValue")
+ settings <- setLabels(settings, "Iterations", "Time (ms)")
+ settings <- setAxis(settings, "Continuous", yAxis)
+ for(size in uniqueSizes){
+ subData3 <- subset(subData2, ChangeSet==size)
+ title <- paste(view, ", ChangeSet: ", size, ", Function: ", concatPhases(phases), sep='')
+ for (extension in config$Extension){
+ fileName <- paste(rootPath, view, "-ChangeSet-", size, "-GroupBy-Tool-", metric, "-", name, ".", extension, sep='')
+ settings <- setTitle(settings, title)
+ savePlot(subData3, settings, phases, fileName)
+ }
+ write.csv(subData3, file = paste(rootPath, view, "-ChangeSet-", size, "-GroupBy-Tool-", metric, "-", name, ".csv", sep=''))
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/scripts/run.py b/scripts/run.py
new file mode 100755
index 0000000000000000000000000000000000000000..cde2a5a27e1163addfe79dd89ad00599b7e62f2b
--- /dev/null
+++ b/scripts/run.py
@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+"""
+@author: Zsolt Kovari, Georg Hinkel
+
+"""
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+try:
+ import ConfigParser
+except ImportError:
+ import configparser as ConfigParser
+import json
+from subprocess import CalledProcessError
+
+BASE_DIRECTORY = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+print("Running benchmark with root directory " + BASE_DIRECTORY)
+
+class JSONObject(object):
+ def __init__(self, d):
+ self.__dict__ = d
+
+
+def build(conf, skip_tests=False):
+ """
+ Builds all solutions
+ """
+ for tool in conf.Tools:
+ config = ConfigParser.ConfigParser()
+ config.read(os.path.join(BASE_DIRECTORY, "solutions", tool, "solution.ini"))
+ set_working_directory("solutions", tool)
+ if skip_tests:
+ subprocess.check_call(config.get('build', 'skipTests'), shell=True)
+ else:
+ subprocess.check_call(config.get('build', 'default'), shell=True)
+
+
+def benchmark(conf):
+ """
+ Runs measurements
+ """
+ header = os.path.join(BASE_DIRECTORY, "output", "header.csv")
+ result_file = os.path.join(BASE_DIRECTORY, "output", "output.csv")
+ if os.path.exists(result_file):
+ os.remove(result_file)
+ shutil.copy(header, result_file)
+ os.environ['Sequences'] = str(conf.Sequences)
+ os.environ['Runs'] = str(conf.Runs)
+ for r in range(0, conf.Runs):
+ os.environ['RunIndex'] = str(r)
+ for tool in conf.Tools:
+ config = ConfigParser.ConfigParser()
+ config.read(os.path.join(BASE_DIRECTORY, "solutions", tool, "solution.ini"))
+ set_working_directory("solutions", tool)
+ os.environ['Tool'] = tool
+ for change_set in conf.ChangeSets:
+ full_change_path = os.path.abspath(os.path.join(BASE_DIRECTORY, "models", change_set))
+ os.environ['ChangeSet'] = change_set
+ os.environ['ChangePath'] = full_change_path
+ for query in conf.Queries:
+ os.environ['Query'] = query
+ print("Running benchmark: tool = " + tool + ", change set = " + change_set +
+ ", query = " + query)
+ try:
+ output = subprocess.check_output(config.get('run', query), shell=True, timeout=conf.Timeout)
+ with open(result_file, "ab") as file:
+ file.write(output)
+ except CalledProcessError as e:
+ print("Program exited with error")
+ except subprocess.TimeoutExpired as e:
+ print("Program reached the timeout set ({0} seconds). The command we executed was '{1}'".format(e.timeout, e.cmd))
+
+
+def clean_dir(*path):
+ dir = os.path.join(BASE_DIRECTORY, *path)
+ if os.path.exists(dir):
+ shutil.rmtree(dir)
+ os.mkdir(dir)
+
+
+def set_working_directory(*path):
+ dir = os.path.join(BASE_DIRECTORY, *path)
+ os.chdir(dir)
+
+
+def visualize():
+ """
+ Visualizes the benchmark results
+ """
+ clean_dir("diagrams")
+ set_working_directory("reporting")
+ subprocess.call(["Rscript", "visualize.R", os.path.join(BASE_DIRECTORY, "config", "reporting.json")])
+
+
+def extract_results():
+ """
+ Extracts the benchmark results
+ """
+ clean_dir("results")
+ set_working_directory("reporting")
+ subprocess.call(["Rscript", "check_results.R"])
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-b", "--build",
+ help="build the project",
+ action="store_true")
+ parser.add_argument("-m", "--measure",
+ help="run the benchmark",
+ action="store_true")
+ parser.add_argument("-s", "--skip-tests",
+ help="skip JUNIT tests",
+ action="store_true")
+ parser.add_argument("-v", "--visualize",
+ help="create visualizations",
+ action="store_true")
+ parser.add_argument("-e", "--extract",
+ help="extract results",
+ action="store_true")
+ parser.add_argument("-t", "--test",
+ help="run test",
+ action="store_true")
+ parser.add_argument("-d", "--debug",
+ help="set debug to true",
+ action="store_true")
+ args = parser.parse_args()
+
+
+ set_working_directory("config")
+ with open("config.json", "r") as config_file:
+ config = json.load(config_file, object_hook = JSONObject)
+
+ if args.debug:
+ os.environ['Debug'] = 'true'
+ if args.build:
+ build(config, args.skip_tests)
+ if args.measure:
+ benchmark(config)
+ if args.test:
+ build(config, False)
+ if args.visualize:
+ visualize()
+ if args.extract:
+ extract_results()
+
+ # if there are no args, execute a full sequence
+ # with the test and the visualization/reporting
+ no_args = all(val==False for val in vars(args).values())
+ if no_args:
+ build(config, False)
+ benchmark(config)
+ visualize()
+ extract_results()