diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..71028dae684d7449a054c33c53adaf25035cf6b3 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 Georg Hinkel + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index fd4621f455bd47c2abdc7e6870660105dcbc1147..7ad53aa94e6c9b3eedf21fb7b33a7a9e43b2e50a 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,51 @@ -# ttc2019-tt2bdd -Free form TTC contest (Truth tables to Binary Decision Diagrams, from ATL Zoo) +# The TTC 2018 Social Media Case + +## Case description + +The `docs/2018_TTC_Live.pdf` file contains the [case description](https://github.com/TransformationToolContest/ttc2018liveContest/raw/master/docs/2018_TTC_Live.pdf). + +## Prerequisites + +* 64-bit operating system +* Python 3.3 or higher +* R + +## Solution Prerequisites + +* NMF: You need to install [.NET Core 2.0](https://www.microsoft.com/net/download/linux-package-manager/ubuntu16-04/sdk-current) + +Add your prerequisites here! + +## Using the framework + +The `scripts` directory contains the `run.py` script. +At a first glance, invoke it without any arguments so that the solution will be built, benchmarked, running times visualized and the results compared to the reference solution's. +One might fine tune the script for the following purposes: +* `run.py -b` -- builds the projects +* `run.py -b -s` -- builds the projects without testing +* `run.py -g` -- generates the instance models +* `run.py -m` -- run the benchmark without building +* `run.py -v` -- visualizes the results of the latest benchmark +* `run.py -e` -- compare results to the reference output. The benchmark shall already been executed using `-m`. +* `run.py -m -e` -- run benchmark without building, then extract and compare results to the reference output +* `run.py -t` -- build the project and run tests (usually unit tests as defined for the given solution) + +The `config` directory contains the configuration for the scripts: +* `config.json` -- configuration for the model generation and the benchmark + * *Note:* the timeout as set in the benchmark configuration (default: 6000 seconds) applies to the gross cumulative runtime of the tool for a given changeset and update sequences. This also includes e.g. Initialization time which is not required by the benchmark framework to be measured. + Timeout is only applied to the solutions' run phase (see `-m` for `run.py`), so it is not applied to e.g. the build phase (see `-b` for `run.py`). +* `reporting.json` -- configuration for the visualization + +### Running the benchmark + +The script runs the benchmark for the given number of runs, for the specified tools and change sequences. + +The benchmark results are stored in a CSV file. The header for the CSV file is stored in the `output/header.csv` file. + +## Reporting and visualization + +Make sure you read the `README.md` file in the `reporting` directory and install all the requirements for R. + +## Implementing the benchmark for a new tool + +To implement a tool, you need to create a new directory in the solutions directory and give it a suitable name. diff --git a/config/config.json b/config/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a3933dd4f9532961176b9a21592c82c8d2376a0f --- /dev/null +++ b/config/config.json @@ -0,0 +1,6 @@ +{ + "Tools": ["ATL"], + "Sequences": 20, + "Runs": 5, + "Timeout": 6000 +} diff --git a/config/reporting.json b/config/reporting.json new file mode 100644 index 0000000000000000000000000000000000000000..cd3e7da23410f437dde08e48b4116497c8d41b26 --- /dev/null +++ b/config/reporting.json @@ -0,0 +1,21 @@ +{ + "Dimensions": { + "X_Dimensions": { + "Size": true, + "Iteration": false + }, + "Groups": { + "View": false, + "Tool": true + } + }, + "Summarize_Functions": [ + { + "Name": "Runtime", + "Phases": [ + "Run" + ] + } + ], + "Extension": ["png", "pdf"] +} diff --git a/expected-results/README.md b/expected-results/README.md new file mode 100644 index 0000000000000000000000000000000000000000..fb3f7304612d6152192bc9eb424f81060c6b677b --- /dev/null +++ b/expected-results/README.md @@ -0,0 +1 @@ +This folder contains the reference result data for the benchmark cases. diff --git a/metamodels/Ecore.ecore b/metamodels/Ecore.ecore new file mode 100644 index 0000000000000000000000000000000000000000..15a3569fbf2ed7be4764087725a9364b9899f9ee --- /dev/null +++ b/metamodels/Ecore.ecore @@ -0,0 +1,526 @@ +<?xml version="1.0" encoding="UTF-8"?> +<ecore:EPackage xmi:version="2.0" xmlns:xmi="http://www.omg.org/XMI" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xmlns:ecore="http://www.eclipse.org/emf/2002/Ecore" name="ecore" nsURI="http://www.eclipse.org/emf/2002/Ecore" nsPrefix="ecore"> + <eClassifiers xsi:type="ecore:EClass" name="EAttribute" eSuperTypes="#//EStructuralFeature"> + <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore"> + <details key="constraints" value="ConsistentTransient"/> + </eAnnotations> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="iD" eType="#//EBoolean"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eAttributeType" lowerBound="1" + eType="#//EDataType" changeable="false" volatile="true" transient="true" derived="true"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EClass" name="EAnnotation" eSuperTypes="#//EModelElement"> + <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore"> + <details key="constraints" value="WellFormedSourceURI"/> + </eAnnotations> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="source" eType="#//EString"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="details" upperBound="-1" + eType="#//EStringToStringMapEntry" containment="true" resolveProxies="false"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eModelElement" eType="#//EModelElement" + transient="true" resolveProxies="false" eOpposite="#//EModelElement/eAnnotations"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="contents" upperBound="-1" + eType="#//EObject" containment="true" resolveProxies="false"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="references" upperBound="-1" + eType="#//EObject"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EClass" name="EClass" eSuperTypes="#//EClassifier"> + <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore"> + <details key="constraints" value="InterfaceIsAbstract AtMostOneID UniqueFeatureNames UniqueOperationSignatures NoCircularSuperTypes WellFormedMapEntryClass ConsistentSuperTypes DisjointFeatureAndOperationSignatures"/> + </eAnnotations> + <eOperations name="isSuperTypeOf" eType="#//EBoolean"> + <eParameters name="someClass" eType="#//EClass"/> + </eOperations> + <eOperations name="getFeatureCount" eType="#//EInt"/> + <eOperations name="getEStructuralFeature" eType="#//EStructuralFeature"> + <eParameters name="featureID" eType="#//EInt"/> + </eOperations> + <eOperations name="getFeatureID" eType="#//EInt"> + <eParameters name="feature" eType="#//EStructuralFeature"/> + </eOperations> + <eOperations name="getEStructuralFeature" eType="#//EStructuralFeature"> + <eParameters name="featureName" eType="#//EString"/> + </eOperations> + <eOperations name="getOperationCount" eType="#//EInt"/> + <eOperations name="getEOperation" eType="#//EOperation"> + <eParameters name="operationID" eType="#//EInt"/> + </eOperations> + <eOperations name="getOperationID" eType="#//EInt"> + <eParameters name="operation" eType="#//EOperation"/> + </eOperations> + <eOperations name="getOverride" eType="#//EOperation"> + <eParameters name="operation" eType="#//EOperation"/> + </eOperations> + <eOperations name="getFeatureType" eType="#//EGenericType"> + <eParameters name="feature" eType="#//EStructuralFeature"/> + </eOperations> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="abstract" eType="#//EBoolean"/> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="interface" eType="#//EBoolean"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eSuperTypes" upperBound="-1" + eType="#//EClass" unsettable="true"> + <eAnnotations source="http://www.eclipse.org/emf/2002/GenModel"> + <details key="suppressedIsSetVisibility" value="true"/> + <details key="suppressedUnsetVisibility" value="true"/> + </eAnnotations> + </eStructuralFeatures> + <eStructuralFeatures xsi:type="ecore:EReference" name="eOperations" upperBound="-1" + eType="#//EOperation" containment="true" resolveProxies="false" eOpposite="#//EOperation/eContainingClass"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eAllAttributes" upperBound="-1" + eType="#//EAttribute" changeable="false" volatile="true" transient="true" + derived="true"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eAllReferences" upperBound="-1" + eType="#//EReference" changeable="false" volatile="true" transient="true" + derived="true"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eReferences" upperBound="-1" + eType="#//EReference" changeable="false" volatile="true" transient="true" + derived="true"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eAttributes" upperBound="-1" + eType="#//EAttribute" changeable="false" volatile="true" transient="true" + derived="true"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eAllContainments" upperBound="-1" + eType="#//EReference" changeable="false" volatile="true" transient="true" + derived="true"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eAllOperations" upperBound="-1" + eType="#//EOperation" changeable="false" volatile="true" transient="true" + derived="true"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eAllStructuralFeatures" + upperBound="-1" eType="#//EStructuralFeature" changeable="false" volatile="true" + transient="true" derived="true"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eAllSuperTypes" upperBound="-1" + eType="#//EClass" changeable="false" volatile="true" transient="true" derived="true"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eIDAttribute" eType="#//EAttribute" + changeable="false" volatile="true" transient="true" derived="true" resolveProxies="false"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eStructuralFeatures" upperBound="-1" + eType="#//EStructuralFeature" containment="true" resolveProxies="false" eOpposite="#//EStructuralFeature/eContainingClass"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eGenericSuperTypes" upperBound="-1" + eType="#//EGenericType" unsettable="true" containment="true" resolveProxies="false"> + <eAnnotations source="http://www.eclipse.org/emf/2002/GenModel"> + <details key="suppressedIsSetVisibility" value="true"/> + <details key="suppressedUnsetVisibility" value="true"/> + </eAnnotations> + </eStructuralFeatures> + <eStructuralFeatures xsi:type="ecore:EReference" name="eAllGenericSuperTypes" + upperBound="-1" eType="#//EGenericType" changeable="false" volatile="true" + transient="true" derived="true"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EClass" name="EClassifier" abstract="true" eSuperTypes="#//ENamedElement"> + <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore"> + <details key="constraints" value="WellFormedInstanceTypeName UniqueTypeParameterNames"/> + </eAnnotations> + <eOperations name="isInstance" eType="#//EBoolean"> + <eParameters name="object" eType="#//EJavaObject"/> + </eOperations> + <eOperations name="getClassifierID" eType="#//EInt"/> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="instanceClassName" eType="#//EString" + volatile="true" unsettable="true"> + <eAnnotations source="http://www.eclipse.org/emf/2002/GenModel"> + <details key="suppressedIsSetVisibility" value="true"/> + <details key="suppressedUnsetVisibility" value="true"/> + </eAnnotations> + </eStructuralFeatures> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="instanceClass" changeable="false" + volatile="true" transient="true" derived="true"> + <eGenericType eClassifier="#//EJavaClass"> + <eTypeArguments/> + </eGenericType> + </eStructuralFeatures> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="defaultValue" eType="#//EJavaObject" + changeable="false" volatile="true" transient="true" derived="true"/> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="instanceTypeName" eType="#//EString" + volatile="true" unsettable="true"> + <eAnnotations source="http://www.eclipse.org/emf/2002/GenModel"> + <details key="suppressedIsSetVisibility" value="true"/> + <details key="suppressedUnsetVisibility" value="true"/> + </eAnnotations> + </eStructuralFeatures> + <eStructuralFeatures xsi:type="ecore:EReference" name="ePackage" eType="#//EPackage" + changeable="false" transient="true" eOpposite="#//EPackage/eClassifiers"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eTypeParameters" upperBound="-1" + eType="#//ETypeParameter" containment="true"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EClass" name="EDataType" eSuperTypes="#//EClassifier"> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="serializable" eType="#//EBoolean" + defaultValueLiteral="true"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EClass" name="EEnum" eSuperTypes="#//EDataType"> + <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore"> + <details key="constraints" value="UniqueEnumeratorNames UniqueEnumeratorLiterals"/> + </eAnnotations> + <eOperations name="getEEnumLiteral" eType="#//EEnumLiteral"> + <eParameters name="name" eType="#//EString"/> + </eOperations> + <eOperations name="getEEnumLiteral" eType="#//EEnumLiteral"> + <eParameters name="value" eType="#//EInt"/> + </eOperations> + <eOperations name="getEEnumLiteralByLiteral" eType="#//EEnumLiteral"> + <eParameters name="literal" eType="#//EString"/> + </eOperations> + <eStructuralFeatures xsi:type="ecore:EReference" name="eLiterals" upperBound="-1" + eType="#//EEnumLiteral" containment="true" resolveProxies="false" eOpposite="#//EEnumLiteral/eEnum"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EClass" name="EEnumLiteral" eSuperTypes="#//ENamedElement"> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="value" eType="#//EInt"/> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="instance" eType="#//EEnumerator" + transient="true"/> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="literal" eType="#//EString"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eEnum" eType="#//EEnum" + changeable="false" transient="true" resolveProxies="false" eOpposite="#//EEnum/eLiterals"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EClass" name="EFactory" eSuperTypes="#//EModelElement"> + <eOperations name="create" eType="#//EObject"> + <eParameters name="eClass" eType="#//EClass"/> + </eOperations> + <eOperations name="createFromString" eType="#//EJavaObject"> + <eParameters name="eDataType" eType="#//EDataType"/> + <eParameters name="literalValue" eType="#//EString"/> + </eOperations> + <eOperations name="convertToString" eType="#//EString"> + <eParameters name="eDataType" eType="#//EDataType"/> + <eParameters name="instanceValue" eType="#//EJavaObject"/> + </eOperations> + <eStructuralFeatures xsi:type="ecore:EReference" name="ePackage" lowerBound="1" + eType="#//EPackage" transient="true" resolveProxies="false" eOpposite="#//EPackage/eFactoryInstance"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EClass" name="EModelElement" abstract="true"> + <eOperations name="getEAnnotation" eType="#//EAnnotation"> + <eParameters name="source" eType="#//EString"/> + </eOperations> + <eStructuralFeatures xsi:type="ecore:EReference" name="eAnnotations" upperBound="-1" + eType="#//EAnnotation" containment="true" resolveProxies="false" eOpposite="#//EAnnotation/eModelElement"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EClass" name="ENamedElement" abstract="true" eSuperTypes="#//EModelElement"> + <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore"> + <details key="constraints" value="WellFormedName"/> + </eAnnotations> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="name" eType="#//EString"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EClass" name="EObject"> + <eOperations name="eClass" eType="#//EClass"/> + <eOperations name="eIsProxy" eType="#//EBoolean"/> + <eOperations name="eResource" eType="#//EResource"/> + <eOperations name="eContainer" eType="#//EObject"/> + <eOperations name="eContainingFeature" eType="#//EStructuralFeature"/> + <eOperations name="eContainmentFeature" eType="#//EReference"/> + <eOperations name="eContents"> + <eGenericType eClassifier="#//EEList"> + <eTypeArguments eClassifier="#//EObject"/> + </eGenericType> + </eOperations> + <eOperations name="eAllContents"> + <eGenericType eClassifier="#//ETreeIterator"> + <eTypeArguments eClassifier="#//EObject"/> + </eGenericType> + </eOperations> + <eOperations name="eCrossReferences"> + <eGenericType eClassifier="#//EEList"> + <eTypeArguments eClassifier="#//EObject"/> + </eGenericType> + </eOperations> + <eOperations name="eGet" eType="#//EJavaObject"> + <eParameters name="feature" eType="#//EStructuralFeature"/> + </eOperations> + <eOperations name="eGet" eType="#//EJavaObject"> + <eParameters name="feature" eType="#//EStructuralFeature"/> + <eParameters name="resolve" eType="#//EBoolean"/> + </eOperations> + <eOperations name="eSet"> + <eParameters name="feature" eType="#//EStructuralFeature"/> + <eParameters name="newValue" eType="#//EJavaObject"/> + </eOperations> + <eOperations name="eIsSet" eType="#//EBoolean"> + <eParameters name="feature" eType="#//EStructuralFeature"/> + </eOperations> + <eOperations name="eUnset"> + <eParameters name="feature" eType="#//EStructuralFeature"/> + </eOperations> + <eOperations name="eInvoke" eType="#//EJavaObject" eExceptions="#//EInvocationTargetException"> + <eParameters name="operation" eType="#//EOperation"/> + <eParameters name="arguments"> + <eGenericType eClassifier="#//EEList"> + <eTypeArguments/> + </eGenericType> + </eParameters> + </eOperations> + </eClassifiers> + <eClassifiers xsi:type="ecore:EClass" name="EOperation" eSuperTypes="#//ETypedElement"> + <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore"> + <details key="constraints" value="UniqueParameterNames UniqueTypeParameterNames NoRepeatingVoid"/> + </eAnnotations> + <eOperations name="getOperationID" eType="#//EInt"/> + <eOperations name="isOverrideOf" eType="#//EBoolean"> + <eParameters name="someOperation" eType="#//EOperation"/> + </eOperations> + <eStructuralFeatures xsi:type="ecore:EReference" name="eContainingClass" eType="#//EClass" + changeable="false" transient="true" resolveProxies="false" eOpposite="#//EClass/eOperations"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eTypeParameters" upperBound="-1" + eType="#//ETypeParameter" containment="true"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eParameters" upperBound="-1" + eType="#//EParameter" containment="true" resolveProxies="false" eOpposite="#//EParameter/eOperation"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eExceptions" upperBound="-1" + eType="#//EClassifier" unsettable="true"> + <eAnnotations source="http://www.eclipse.org/emf/2002/GenModel"> + <details key="suppressedIsSetVisibility" value="true"/> + <details key="suppressedUnsetVisibility" value="true"/> + </eAnnotations> + </eStructuralFeatures> + <eStructuralFeatures xsi:type="ecore:EReference" name="eGenericExceptions" upperBound="-1" + eType="#//EGenericType" unsettable="true" containment="true" resolveProxies="false"> + <eAnnotations source="http://www.eclipse.org/emf/2002/GenModel"> + <details key="suppressedIsSetVisibility" value="true"/> + <details key="suppressedUnsetVisibility" value="true"/> + </eAnnotations> + </eStructuralFeatures> + </eClassifiers> + <eClassifiers xsi:type="ecore:EClass" name="EPackage" eSuperTypes="#//ENamedElement"> + <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore"> + <details key="constraints" value="WellFormedNsURI WellFormedNsPrefix UniqueSubpackageNames UniqueClassifierNames UniqueNsURIs"/> + </eAnnotations> + <eOperations name="getEClassifier" eType="#//EClassifier"> + <eParameters name="name" eType="#//EString"/> + </eOperations> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="nsURI" eType="#//EString"/> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="nsPrefix" eType="#//EString"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eFactoryInstance" lowerBound="1" + eType="#//EFactory" transient="true" resolveProxies="false" eOpposite="#//EFactory/ePackage"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eClassifiers" upperBound="-1" + eType="#//EClassifier" containment="true" eOpposite="#//EClassifier/ePackage"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eSubpackages" upperBound="-1" + eType="#//EPackage" containment="true" eOpposite="#//EPackage/eSuperPackage"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eSuperPackage" eType="#//EPackage" + changeable="false" transient="true" eOpposite="#//EPackage/eSubpackages"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EClass" name="EParameter" eSuperTypes="#//ETypedElement"> + <eStructuralFeatures xsi:type="ecore:EReference" name="eOperation" eType="#//EOperation" + changeable="false" transient="true" resolveProxies="false" eOpposite="#//EOperation/eParameters"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EClass" name="EReference" eSuperTypes="#//EStructuralFeature"> + <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore"> + <details key="constraints" value="ConsistentOpposite SingleContainer ConsistentKeys ConsistentUnique ConsistentContainer"/> + </eAnnotations> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="containment" eType="#//EBoolean"/> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="container" eType="#//EBoolean" + changeable="false" volatile="true" transient="true" derived="true"/> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="resolveProxies" eType="#//EBoolean" + defaultValueLiteral="true"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eOpposite" eType="#//EReference"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eReferenceType" lowerBound="1" + eType="#//EClass" changeable="false" volatile="true" transient="true" derived="true"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eKeys" upperBound="-1" + eType="#//EAttribute"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EClass" name="EStructuralFeature" abstract="true" + eSuperTypes="#//ETypedElement"> + <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore"> + <details key="constraints" value="ValidDefaultValueLiteral"/> + </eAnnotations> + <eOperations name="getFeatureID" eType="#//EInt"/> + <eOperations name="getContainerClass"> + <eGenericType eClassifier="#//EJavaClass"> + <eTypeArguments/> + </eGenericType> + </eOperations> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="changeable" eType="#//EBoolean" + defaultValueLiteral="true"/> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="volatile" eType="#//EBoolean"/> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="transient" eType="#//EBoolean"/> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="defaultValueLiteral" eType="#//EString"/> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="defaultValue" eType="#//EJavaObject" + changeable="false" volatile="true" transient="true" derived="true"/> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="unsettable" eType="#//EBoolean"/> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="derived" eType="#//EBoolean"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eContainingClass" eType="#//EClass" + changeable="false" transient="true" resolveProxies="false" eOpposite="#//EClass/eStructuralFeatures"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EClass" name="ETypedElement" abstract="true" eSuperTypes="#//ENamedElement"> + <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore"> + <details key="constraints" value="ValidLowerBound ValidUpperBound ConsistentBounds ValidType"/> + </eAnnotations> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="ordered" eType="#//EBoolean" + defaultValueLiteral="true"/> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="unique" eType="#//EBoolean" + defaultValueLiteral="true"/> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="lowerBound" eType="#//EInt"/> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="upperBound" eType="#//EInt" + defaultValueLiteral="1"/> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="many" eType="#//EBoolean" + changeable="false" volatile="true" transient="true" derived="true"/> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="required" eType="#//EBoolean" + changeable="false" volatile="true" transient="true" derived="true"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eType" eType="#//EClassifier" + volatile="true" unsettable="true"> + <eAnnotations source="http://www.eclipse.org/emf/2002/GenModel"> + <details key="suppressedIsSetVisibility" value="true"/> + <details key="suppressedUnsetVisibility" value="true"/> + </eAnnotations> + </eStructuralFeatures> + <eStructuralFeatures xsi:type="ecore:EReference" name="eGenericType" eType="#//EGenericType" + volatile="true" unsettable="true" containment="true" resolveProxies="false"> + <eAnnotations source="http://www.eclipse.org/emf/2002/GenModel"> + <details key="suppressedIsSetVisibility" value="true"/> + <details key="suppressedUnsetVisibility" value="true"/> + </eAnnotations> + </eStructuralFeatures> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EBigDecimal" instanceClassName="java.math.BigDecimal"> + <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData"> + <details key="baseType" value="http://www.w3.org/2001/XMLSchema#decimal"/> + </eAnnotations> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EBigInteger" instanceClassName="java.math.BigInteger"> + <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData"> + <details key="baseType" value="http://www.w3.org/2001/XMLSchema#integer"/> + </eAnnotations> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EBoolean" instanceClassName="boolean"> + <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData"> + <details key="baseType" value="http://www.w3.org/2001/XMLSchema#boolean"/> + </eAnnotations> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EBooleanObject" instanceClassName="java.lang.Boolean"> + <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData"> + <details key="baseType" value="EBoolean"/> + <details key="name" value="EBoolean:Object"/> + </eAnnotations> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EByte" instanceClassName="byte"> + <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData"> + <details key="baseType" value="http://www.w3.org/2001/XMLSchema#byte"/> + </eAnnotations> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EByteArray" instanceClassName="byte[]"> + <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData"> + <details key="baseType" value="http://www.w3.org/2001/XMLSchema#hexBinary"/> + </eAnnotations> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EByteObject" instanceClassName="java.lang.Byte"> + <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData"> + <details key="baseType" value="EByte"/> + <details key="name" value="EByte:Object"/> + </eAnnotations> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EChar" instanceClassName="char"/> + <eClassifiers xsi:type="ecore:EDataType" name="ECharacterObject" instanceClassName="java.lang.Character"> + <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData"> + <details key="baseType" value="EChar"/> + <details key="name" value="EChar:Object"/> + </eAnnotations> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EDate" instanceClassName="java.util.Date"/> + <eClassifiers xsi:type="ecore:EDataType" name="EDiagnosticChain" instanceClassName="org.eclipse.emf.common.util.DiagnosticChain" + serializable="false"/> + <eClassifiers xsi:type="ecore:EDataType" name="EDouble" instanceClassName="double"> + <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData"> + <details key="baseType" value="http://www.w3.org/2001/XMLSchema#double"/> + </eAnnotations> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EDoubleObject" instanceClassName="java.lang.Double"> + <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData"> + <details key="baseType" value="EDouble"/> + <details key="name" value="EDouble:Object"/> + </eAnnotations> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EEList" instanceClassName="org.eclipse.emf.common.util.EList" + serializable="false"> + <eTypeParameters name="E"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EEnumerator" instanceClassName="org.eclipse.emf.common.util.Enumerator" + serializable="false"/> + <eClassifiers xsi:type="ecore:EDataType" name="EFeatureMap" instanceClassName="org.eclipse.emf.ecore.util.FeatureMap" + serializable="false"/> + <eClassifiers xsi:type="ecore:EDataType" name="EFeatureMapEntry" instanceClassName="org.eclipse.emf.ecore.util.FeatureMap$Entry" + serializable="false"/> + <eClassifiers xsi:type="ecore:EDataType" name="EFloat" instanceClassName="float"> + <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData"> + <details key="baseType" value="http://www.w3.org/2001/XMLSchema#float"/> + </eAnnotations> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EFloatObject" instanceClassName="java.lang.Float"> + <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData"> + <details key="baseType" value="EFloat"/> + <details key="name" value="EFloat:Object"/> + </eAnnotations> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EInt" instanceClassName="int"> + <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData"> + <details key="baseType" value="http://www.w3.org/2001/XMLSchema#int"/> + </eAnnotations> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EIntegerObject" instanceClassName="java.lang.Integer"> + <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData"> + <details key="baseType" value="EInt"/> + <details key="name" value="EInt:Object"/> + </eAnnotations> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EJavaClass" instanceClassName="java.lang.Class"> + <eTypeParameters name="T"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EJavaObject" instanceClassName="java.lang.Object"/> + <eClassifiers xsi:type="ecore:EDataType" name="ELong" instanceClassName="long"> + <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData"> + <details key="baseType" value="http://www.w3.org/2001/XMLSchema#long"/> + </eAnnotations> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="ELongObject" instanceClassName="java.lang.Long"> + <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData"> + <details key="baseType" value="ELong"/> + <details key="name" value="ELong:Object"/> + </eAnnotations> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EMap" instanceClassName="java.util.Map" + serializable="false"> + <eTypeParameters name="K"/> + <eTypeParameters name="V"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EResource" instanceClassName="org.eclipse.emf.ecore.resource.Resource" + serializable="false"/> + <eClassifiers xsi:type="ecore:EDataType" name="EResourceSet" instanceClassName="org.eclipse.emf.ecore.resource.ResourceSet" + serializable="false"/> + <eClassifiers xsi:type="ecore:EDataType" name="EShort" instanceClassName="short"> + <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData"> + <details key="baseType" value="http://www.w3.org/2001/XMLSchema#short"/> + </eAnnotations> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EShortObject" instanceClassName="java.lang.Short"> + <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData"> + <details key="baseType" value="EShort"/> + <details key="name" value="EShort:Object"/> + </eAnnotations> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EString" instanceClassName="java.lang.String"> + <eAnnotations source="http:///org/eclipse/emf/ecore/util/ExtendedMetaData"> + <details key="baseType" value="http://www.w3.org/2001/XMLSchema#string"/> + </eAnnotations> + </eClassifiers> + <eClassifiers xsi:type="ecore:EClass" name="EStringToStringMapEntry" instanceClassName="java.util.Map$Entry"> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="key" eType="#//EString"/> + <eStructuralFeatures xsi:type="ecore:EAttribute" name="value" eType="#//EString"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="ETreeIterator" instanceClassName="org.eclipse.emf.common.util.TreeIterator" + serializable="false"> + <eTypeParameters name="E"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EClass" name="EGenericType"> + <eAnnotations source="http://www.eclipse.org/emf/2002/Ecore"> + <details key="constraints" value="ConsistentType ConsistentBounds ConsistentArguments"/> + </eAnnotations> + <eOperations name="isInstance" eType="#//EBoolean"> + <eParameters name="object" eType="#//EJavaObject"/> + </eOperations> + <eStructuralFeatures xsi:type="ecore:EReference" name="eUpperBound" eType="#//EGenericType" + containment="true" resolveProxies="false"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eTypeArguments" upperBound="-1" + eType="#//EGenericType" containment="true" resolveProxies="false"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eRawType" lowerBound="1" + eType="#//EClassifier" changeable="false" transient="true" derived="true"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eLowerBound" eType="#//EGenericType" + containment="true" resolveProxies="false"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eTypeParameter" eType="#//ETypeParameter" + resolveProxies="false"/> + <eStructuralFeatures xsi:type="ecore:EReference" name="eClassifier" eType="#//EClassifier"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EClass" name="ETypeParameter" eSuperTypes="#//ENamedElement"> + <eStructuralFeatures xsi:type="ecore:EReference" name="eBounds" upperBound="-1" + eType="#//EGenericType" containment="true" resolveProxies="false"/> + </eClassifiers> + <eClassifiers xsi:type="ecore:EDataType" name="EInvocationTargetException" instanceClassName="java.lang.reflect.InvocationTargetException" + serializable="false"/> +</ecore:EPackage> diff --git a/models/.gitignore b/models/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..91711cb0f57d823ee9a3b65682b4b01a706b10f2 --- /dev/null +++ b/models/.gitignore @@ -0,0 +1 @@ +/1024/ diff --git a/output/header.csv b/output/header.csv new file mode 100644 index 0000000000000000000000000000000000000000..3db9cb29150270a7ae23a5e246199df7a541efd1 --- /dev/null +++ b/output/header.csv @@ -0,0 +1 @@ +Tool;Size;RunIndex;Iteration;PhaseName;MetricName;MetricValue diff --git a/reporting/README.md b/reporting/README.md new file mode 100644 index 0000000000000000000000000000000000000000..557cc2dd88a25c3fbc88c21f817f1f5bd6fcc00b --- /dev/null +++ b/reporting/README.md @@ -0,0 +1,70 @@ +# TTC 2017 Smart Grids Benchmark reporting tools + +## Dependencies + +1. Install R: the `r-base`, `r-base-dev` packages are required. The code is tested with R 3.0 (you can check your version with the `R --version` command). + + ```bash + sudo apt-get install -y r-base r-base-dev + sudo apt-get install -y libssl-dev + ``` + +1. Install the required R packages by issuing the following command: + + ```bash + cd reporting/ + sudo R -f install.R + ``` + + As a consequence, the `ggplot2`, `jsonlite`, `plyr` packages will be installed. + +## Usage + +1. Adjust the configuration parameters in the `config/reporting.json` file. +1. Use the `scripts/run.py -v` command. +1. Alternatively, execute the following command from the `reporting` directory: + + `Rscript generate_diagrams.R` + +### Configuration + +* An example of the configuration file can be seen below: + + ```json +{ + "Dimensions": { + "X_Dimensions": { + "Size": true, + "Iteration": true + }, + "Groups": { + "Query": true, + "Tool": true + } + }, + "Summarize_Functions": [ + { + "Phases": [ + "check", + "transformation" + ], + "Metric": "time", + "Y_Label": "Time (ms)", + "Y_Axis_Scale": -6 + }, + { + "Phases": [ + "result" + ], + "Metric": "scalar", + "Y_Label": "Invalid elements", + "Y_Axis_Scale": 0 + } + ], + "Extension": ["png", "pdf"] +} + ``` + +* An arbitrary number of functions can be declared. + +* The dimension and group parameters can be changed independently from each other, which means for example every variable can be `true` at the same time. As a consequence, more diagrams will be generated. diff --git a/reporting/check_results.R b/reporting/check_results.R new file mode 100644 index 0000000000000000000000000000000000000000..e13d43260247e6ae5b383b24bb72a3d3f6097f18 --- /dev/null +++ b/reporting/check_results.R @@ -0,0 +1,25 @@ +expected = read.csv2("../expected-results/results.csv") +actual = subset(read.csv2("../output/output.csv"), MetricName=="Elements") + +tools = unique(actual$Tool) + +for (tool in tools) { + tool_data = subset(actual, Tool==tool) + queries = unique(tool_data$View) + for (query in queries) { + query_data = subset(tool_data, View==query) + query_n = length(row.names(query_data)) + for (i in 1:query_n) { + query.row = query_data[i,] + expected.row = subset(expected, ChangeSet==query.row$ChangeSet & View==query & Iteration==query.row$Iteration) + + if (length(as.character(expected.row$MetricValue)) > 0) { + if (as.character(query.row$MetricValue) != as.character(expected.row$MetricValue)) { + print(paste(tool, "is wrong. Was ", query.row$MetricValue, "but expected", expected.row$MetricValue, "for change set", query.row$ChangeSet, "query", query, "iteration", query.row$Iteration)) + } + } else { + print(paste("Warning:", tool, "produced the result", query.row$MetricValue, "but expected result is unavailable for change set", query.row$ChangeSet, "query", query, "iteration", query.row$Iteration)) + } + } + } +} diff --git a/reporting/constants.R b/reporting/constants.R new file mode 100644 index 0000000000000000000000000000000000000000..481367ff8dcdf82b73b610951ee646ade8c86d82 --- /dev/null +++ b/reporting/constants.R @@ -0,0 +1,30 @@ +resultsPath <- "../output/output.csv" + +# y-axis scaling +scaleDivisor <- 10**6 + +# output folder +rootPath <- c("../diagrams/") + +# plot settings +pointSize = 3 +lineSize = 1 +yLabel <- "Time (ms)" +yAxis <- "Log10" + +labels <- list("1" = "1\n ", + "2" = "2\n ", + "4" = "4\n ", + "8" = "8\n ", + "16"= "16\n ", + "32"= "32\n ", + "64" = "64\n ", + "128" = "128\n ", + "256" = "256\n ", + "512" = "512\n ", + "1024" = "1024\n ", + "2048" = "2048\n ", + "4096" = "4096\n ", + "8192" = "8192\n ", + "16384" = "16384\n " + ) diff --git a/reporting/extract_results.R b/reporting/extract_results.R new file mode 100644 index 0000000000000000000000000000000000000000..b3b0eeadf4b7f285f593327679daa24428e67438 --- /dev/null +++ b/reporting/extract_results.R @@ -0,0 +1,35 @@ +library("plyr", quietly=T, verbose=F, warn.conflicts=FALSE) +source("constants.R") +results <-read.csv2(resultsPath, header=TRUE) + +changes <- unique(results$ChangeSet) +metricName <- "elements" + +for (change in changes){ + subData <- subset(results, ChangeSet == change) + queries <- unique(subData$Query) + for (query in queries){ + fileName <- paste("../results/",change, "-", query, ".csv", sep="") + if (file.exists(fileName)){ + file.remove(fileName) + } + + firstRow <- list("Size") + subData2 <- subset(subData, Query == query) + iterations <- unique(subData2$Iteration) + for (iter in iterations){ + firstRow <- paste(firstRow, iter, sep="\t") + } + write(firstRow, fileName, append=TRUE) + sizes <- unique(subData2$Size) + for (size in sizes){ + row <- list(size) + for (iter in iterations){ + metricValue <- subset(subData2, Size==size & Iteration==iter & MetricName==metricName & RunIndex==1)$MetricValue + row <- paste(row, metricValue, sep="\t") + } + write(row, fileName, append=TRUE) + } + + } +} diff --git a/reporting/functions.R b/reporting/functions.R new file mode 100644 index 0000000000000000000000000000000000000000..2c1663533445862d6413e1e2c1d16afdf506b788 --- /dev/null +++ b/reporting/functions.R @@ -0,0 +1,125 @@ +source("theme.R") + +savePlot <-function(results, settings, phases, fileName){ + phases <- unlist(phases) + first <- TRUE + for(phase in phases){ + + if (first == TRUE){ + merged <- subset(results, PhaseName == phase) # selection on phase + first <- FALSE + } + else{ + merged <- rbind(merged, subset(results, PhaseName == phase)) # merge back + } + } + + if (nrow(merged) == 0) { + return() + } + if (settings@xDimension == "ChangeSet"){ + # summarise the iterations + data <- ddply(merged, c("ChangeSet", "View", "Tool", "RunIndex", "MetricName"), + summarize, MetricValue=sum(MetricValue)) + data <- ddply(data, c("ChangeSet", "View", "Tool", "MetricName"), + summarize, MetricValue=mean(MetricValue)) + } + else if(settings@xDimension == "Iteration"){ + data <- ddply(merged, c("View", "Tool", "ChangeSet", "RunIndex", "MetricName", "Iteration"), + summarize, MetricValue=sum(MetricValue)) + data <- ddply(data, c("View", "Tool", "ChangeSet", "MetricName", "Iteration"), + summarize, MetricValue=median(MetricValue)) + } + else { + return() + } + artifacts <- unique(data[[settings@xDimension]]) + minValue <- min(data$MetricValue) + maxValue <- max(data$MetricValue) + + xLabels <- c(artifacts) + plot <- ggplot(data,aes_string(x = settings@xDimension, y = settings@yDimension)) + + geom_line(aes_string(group = settings@group, colour=settings@group), size=lineSize) + + geom_point(aes_string(shape = settings@group, colour=settings@group), size=pointSize) + + scale_shape_manual(values=1:nlevels(data[[settings@group]])) + + ylab(settings@yLabel) + + ggtitle(label = settings@title) + + bwTheme + + if (settings@xAxis == "Continuous"){ + plot <- plot + scale_x_continuous(breaks = c(artifacts)) + } + else if (settings@xAxis == "Log2"){ + plot <- plot + scale_x_log10(breaks = c(artifacts), labels = xLabels) + } + if (minValue == 0){ + print("The minimum metricvalue equals with 0. The plot cannot be generated.") + return() + } + if (settings@yAxis == "Continuous"){ + plot <- plot + scale_y_continuous(breaks = seq(minValue, maxValue, by=round(maxValue/5,0)), + labels = seq(minValue, maxValue, by=round(maxValue/5,0))) + } + else if (settings@yAxis == "Log10"){ + plot <- plot + scale_y_log10(breaks = 10^seq(round(log10(minValue)), round(log10(maxValue)), by=1), + labels = 10^seq(round(log10(minValue)), round(log10(maxValue)), by=1)) + } + ggsave(plot,filename = fileName, width=10, height=5, dpi=192) + print(fileName) + + +} + +validPhase <- function(results, functions){ + uniquePhases <- unique(results$PhaseName) + for(func in functions){ + for(phase in func){ + if(phase %in% uniquePhases == FALSE){ + return(FALSE) + } + } + } + return(TRUE) +} + +validMetric <- function(results, metric){ + if (metric %in% levels(results$MetricName)){ + return(TRUE) + } + return(FALSE) +} + +createFolders <- function(rootPath, subFolders){ + if (file.exists(rootPath) == FALSE){ + dir.create(rootPath) + } + for (folder in subFolders){ + path <- paste(rootPath, folder, sep='') + if (file.exists(path) == FALSE){ + dir.create(path) + } + } +} + +concatPhases <- function(phases){ + phases <- unlist(phases) + first <- TRUE + for(p in phases){ + if(first){ + merged <- p + first <- FALSE + } + else{ + merged <- paste(merged, p, sep='+') + } + } + return(merged) +} + +getXLabels <- function(artifacts){ + ticks <- c() + for(size in artifacts){ + ticks <- c(ticks, labels[[as.character(size)]]) + } + return(ticks) +} diff --git a/reporting/install.R b/reporting/install.R new file mode 100644 index 0000000000000000000000000000000000000000..5fc125685bb7385360101d776aa03928cd8342bc --- /dev/null +++ b/reporting/install.R @@ -0,0 +1,5 @@ +install.packages("ggplot2", repos="http://cran.rstudio.com/"); +install.packages("jsonlite", repos="http://cran.rstudio.com/"); +install.packages("devtools", repos="http://cran.rstudio.com/") +require(devtools) +install_version("plyr", version = "1.8.1", repos = "http://cran.rstudio.com/") diff --git a/reporting/memory.r b/reporting/memory.r new file mode 100644 index 0000000000000000000000000000000000000000..a4a8c905777c92d63588a18858245f222408ccde --- /dev/null +++ b/reporting/memory.r @@ -0,0 +1,42 @@ +library("ggplot2") +library("plyr") + +results <- read.csv2("../output/output.csv", row.names = NULL) + +testset = ddply(results, c("Tool", "RunIndex", "ChangeSet", "View"), MetricValue=max(MetricValue)) +testset = ddply(testset, c("Tool", "ChangeSet", "View"), summarize, Memory=mean(MetricValue)/(1024*1024)) + +pointSize = 3 +lineSize = 1 + +bwTheme <- theme(text=element_text(family="Helvetica", size=16), + panel.background = element_rect(fill="#FFFFFF"), + legend.position="bottom", + legend.direction="horizontal", + panel.grid.major = element_line(size=0.3, colour="#333333"), + panel.grid.minor = element_line(size=0.15, colour="#CCCCCC"), + axis.text.x = element_text(colour="black"), + axis.text.y = element_text(colour="black"), + legend.position="right" +) + +for (query in unique(testset$View)) { + + data = subset(testset, View==query) + + sizes <- unique(data$ChangeSet) + minValue <- min(data$Memory) + maxValue <- max(data$Memory) + +plot <- ggplot(data, aes_string(x="ChangeSet",y="Memory")) +plot <- plot + geom_line(aes_string(group="Tool",colour="Tool"),size=lineSize) +plot <- plot + geom_point(aes_string(shape="Tool", colour="Tool"), size=pointSize) +plot <- plot + scale_shape_manual(values=1:4) + ylab("Memory (Mbyte)") +plot <- plot + xlab("ChangeSet") +plot <- plot + bwTheme +plot <- plot + scale_y_log10(breaks = 10^seq(round(log10(minValue)), round(log10(maxValue)), by=1), + labels = 10^seq(round(log10(minValue)), round(log10(maxValue)), by=1)) +plot <- plot + ggtitle(paste(query, ", working set", sep="")) + +ggsave(plot, filename=paste(query, "-memory.pdf", sep=""), width=7, height=4, dpi=192) +} \ No newline at end of file diff --git a/reporting/plot.R b/reporting/plot.R new file mode 100644 index 0000000000000000000000000000000000000000..ae36a205abcc8c6942e4fa69f06993cb4b89e665 --- /dev/null +++ b/reporting/plot.R @@ -0,0 +1,93 @@ +PlotSettings <- setClass( + "PlotSettings", + + slots = c( + xDimension = "character", + yDimension = "character", + group = "character", + xLabel = "character", + yLabel = "character", + title = "character", + xAxis = "character", + yAxis = "character" + ), + + prototype=list( + xDimension = "", + yDimension = "", + group = "", + xLabel = "", + yLabel = "", + title = "", + xAxis = "", + yAxis = "" + ) + ) + +setGeneric(name="setDimensions", + def=function(theObject, xDimension, yDimension){ + standardGeneric("setDimensions") + } +) + +setMethod(f="setDimensions", + signature="PlotSettings", + definition=function(theObject, xDimension, yDimension){ + theObject@xDimension <- xDimension + theObject@yDimension <- yDimension + return(theObject) + }) + +setGeneric(name="setGroup", + def=function(theObject, group){ + standardGeneric("setGroup") + } +) + +setMethod(f="setGroup", + signature="PlotSettings", + definition=function(theObject, group){ + theObject@group <- group + return(theObject) + }) + +setGeneric(name="setTitle", + def=function(theObject, title){ + standardGeneric("setTitle") + } +) + +setMethod(f="setTitle", + signature="PlotSettings", + definition=function(theObject, title){ + theObject@title <- title + return(theObject) + }) + +setGeneric(name="setLabels", + def=function(theObject, xLabel, yLabel){ + standardGeneric("setLabels") + } + ) + +setMethod(f="setLabels", + signature="PlotSettings", + definition=function(theObject, xLabel, yLabel){ + theObject@xLabel <- xLabel + theObject@yLabel <- yLabel + return(theObject) + }) + +setGeneric(name="setAxis", + def=function(theObject, xAxis, yAxis){ + standardGeneric("setAxis") + } +) + +setMethod(f="setAxis", + signature="PlotSettings", + definition=function(theObject, xAxis, yAxis){ + theObject@xAxis <- xAxis + theObject@yAxis <- yAxis + return(theObject) + }) diff --git a/reporting/theme.R b/reporting/theme.R new file mode 100644 index 0000000000000000000000000000000000000000..fdafd77f3caf2801718523d59cd35483d7523372 --- /dev/null +++ b/reporting/theme.R @@ -0,0 +1,9 @@ +bwTheme <- theme(text=element_text(family="Helvetica", size=16), + panel.background = element_rect(fill="#FFFFFF"), + legend.position="bottom", + legend.direction="horizontal", + panel.grid.major = element_line(size=0.3, colour="#333333"), + panel.grid.minor = element_line(size=0.15, colour="#CCCCCC"), + axis.text.x = element_text(colour="black"), + axis.text.y = element_text(colour="black") +) \ No newline at end of file diff --git a/reporting/visualize.R b/reporting/visualize.R new file mode 100644 index 0000000000000000000000000000000000000000..50d2b94ded3976de5833f04b52ee635196763dc0 --- /dev/null +++ b/reporting/visualize.R @@ -0,0 +1,116 @@ +library("jsonlite", quietly=T, verbose=F, warn.conflicts=FALSE) +library("ggplot2",quietly=T, verbose=F, warn.conflicts=FALSE) +library("plyr", quietly=T, verbose=F, warn.conflicts=FALSE) +source("functions.R") +source("plot.R") +source("constants.R") + +args <- commandArgs(trailingOnly = TRUE) +configPath <- args[1] + +results <-read.csv2(resultsPath, header=TRUE, row.names = NULL) +results$ChangeSet = as.factor(results$ChangeSet) + +config <- fromJSON(configPath) + +if (validPhase(results, config$Summarize_Functions$Phases) == FALSE){ + print("Non existing phasename provided!") + quit() +} + +index <- 0 +settings <- PlotSettings() +uniqueScenarios <- unique(results$ChangeSet) + +for(row in 1:nrow(config$Summarize_Functions)){ + phases <- config$Summarize_Functions[row,]$Phases + name <- config$Summarize_Functions[row,]$Name + index <- index + 1 + for(scenario in uniqueScenarios){ + metric <- "Time" + subData1 <- subset(results, ChangeSet==scenario & MetricName == metric) + subData1$MetricValue <- as.numeric(as.character(subData1$MetricValue)) * (10**-6) + + if (config$Dimensions$Groups$View){ + uniqueTools <- unique(subData1$Tool) + settings <- setGroup(settings, "View") + for(tool in uniqueTools){ + subData2 <- subset(subData1, Tool==tool) + + if (config$Dimensions$X_Dimensions$ChangeSet){ + title <- paste(tool, ", ", scenario, ", Function: ", concatPhases(phases), sep='') + + settings <- setTitle(settings, title) + settings <- setDimensions(settings, "ChangeSet", "MetricValue") + settings <- setLabels(settings, "ChangeSet", "Time (ms)") + settings <- setAxis(settings, "Factor", yAxis) + for (extension in config$Extension){ + fileName <- paste(rootPath, scenario, "-", tool, "-GroupBy-View-", metric, "-", name, ".", extension, sep='') + savePlot(subData2, settings, phases, fileName) + } + write.csv(subData2, file = paste(rootPath, scenario, "-", tool, "-GroupBy-View-", metric, "-", name, ".csv", sep='')) + } + + if (config$Dimensions$X_Dimensions$Iteration){ + uniqueChangeSets <-unique(subData2$ChangeSet) + for(changeSet in uniqueChangeSets){ + subData3 <- subset(subData2, ChangeSet==changeSet) + title <- paste(tool, ", ChangeSet: ", changeSet, ", Function: ", concatPhases(phases), sep='') + + settings <- setTitle(settings, title) + settings <- setDimensions(settings, "Iteration", "MetricValue") + settings <- setLabels(settings, "Iterations", "Time (ms)") + settings <- setAxis(settings, "Continuous", yAxis) + for (extension in config$Extension){ + fileName <- paste(rootPath, scenario, "-", tool, "-changeSet-", changeSet, "-GroupBy-View-", metric, "-", name, ".", extension, sep='') + savePlot(subData3, settings, phases, fileName) + } + write.csv(subData3, file = paste(rootPath, scenario, "-", tool, "-ChangeSet-", changeSet, "-GroupBy-View-", metric, "-", name, ".csv", sep='')) + } + } + } + } + } + + if (config$Dimensions$Groups$Tool){ + metric <- "Time" + subData1 <- subset(results, MetricName == metric) + subData1$MetricValue <- as.numeric(as.character(subData1$MetricValue)) * (10**-6) + + uniqueViews <- unique(subData1$View) + settings <- setGroup(settings, "Tool") + for(view in uniqueViews){ + subData2 <- subset(subData1, View==view) + + if (config$Dimensions$X_Dimensions$ChangeSet){ + title <- paste(view, ", Function: ", concatPhases(phases), sep='') + settings <- setTitle(settings, title) + settings <- setDimensions(settings, "ChangeSet", "MetricValue") + settings <- setLabels(settings, "ChangeSet", "Time (ms)") + settings <- setAxis(settings, "Discrete", yAxis) + for (extension in config$Extension){ + fileName <- paste(rootPath, view, "-GroupBy-Tool-",metric, "-", name, ".", extension, sep='') + savePlot(subData2, settings, phases, fileName) + } + write.csv(ddply(subData2, c("Tool", "ChangeSet"), summarise, N=length(MetricValue), mean=mean(MetricValue), sd=sd(MetricValue)), file = paste(rootPath, view, "-GroupBy-Tool-",metric, "-", name, ".csv", sep='')) + } + + if (config$Dimensions$X_Dimensions$Iteration){ + uniqueSizes <-unique(subData2$ChangeSet) + settings <- setDimensions(settings, "Iteration", "MetricValue") + settings <- setLabels(settings, "Iterations", "Time (ms)") + settings <- setAxis(settings, "Continuous", yAxis) + for(size in uniqueSizes){ + subData3 <- subset(subData2, ChangeSet==size) + title <- paste(view, ", ChangeSet: ", size, ", Function: ", concatPhases(phases), sep='') + for (extension in config$Extension){ + fileName <- paste(rootPath, view, "-ChangeSet-", size, "-GroupBy-Tool-", metric, "-", name, ".", extension, sep='') + settings <- setTitle(settings, title) + savePlot(subData3, settings, phases, fileName) + } + write.csv(subData3, file = paste(rootPath, view, "-ChangeSet-", size, "-GroupBy-Tool-", metric, "-", name, ".csv", sep='')) + } + } + } + } +} \ No newline at end of file diff --git a/scripts/run.py b/scripts/run.py new file mode 100755 index 0000000000000000000000000000000000000000..cde2a5a27e1163addfe79dd89ad00599b7e62f2b --- /dev/null +++ b/scripts/run.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +""" +@author: Zsolt Kovari, Georg Hinkel + +""" +import argparse +import os +import shutil +import subprocess +import sys +try: + import ConfigParser +except ImportError: + import configparser as ConfigParser +import json +from subprocess import CalledProcessError + +BASE_DIRECTORY = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +print("Running benchmark with root directory " + BASE_DIRECTORY) + +class JSONObject(object): + def __init__(self, d): + self.__dict__ = d + + +def build(conf, skip_tests=False): + """ + Builds all solutions + """ + for tool in conf.Tools: + config = ConfigParser.ConfigParser() + config.read(os.path.join(BASE_DIRECTORY, "solutions", tool, "solution.ini")) + set_working_directory("solutions", tool) + if skip_tests: + subprocess.check_call(config.get('build', 'skipTests'), shell=True) + else: + subprocess.check_call(config.get('build', 'default'), shell=True) + + +def benchmark(conf): + """ + Runs measurements + """ + header = os.path.join(BASE_DIRECTORY, "output", "header.csv") + result_file = os.path.join(BASE_DIRECTORY, "output", "output.csv") + if os.path.exists(result_file): + os.remove(result_file) + shutil.copy(header, result_file) + os.environ['Sequences'] = str(conf.Sequences) + os.environ['Runs'] = str(conf.Runs) + for r in range(0, conf.Runs): + os.environ['RunIndex'] = str(r) + for tool in conf.Tools: + config = ConfigParser.ConfigParser() + config.read(os.path.join(BASE_DIRECTORY, "solutions", tool, "solution.ini")) + set_working_directory("solutions", tool) + os.environ['Tool'] = tool + for change_set in conf.ChangeSets: + full_change_path = os.path.abspath(os.path.join(BASE_DIRECTORY, "models", change_set)) + os.environ['ChangeSet'] = change_set + os.environ['ChangePath'] = full_change_path + for query in conf.Queries: + os.environ['Query'] = query + print("Running benchmark: tool = " + tool + ", change set = " + change_set + + ", query = " + query) + try: + output = subprocess.check_output(config.get('run', query), shell=True, timeout=conf.Timeout) + with open(result_file, "ab") as file: + file.write(output) + except CalledProcessError as e: + print("Program exited with error") + except subprocess.TimeoutExpired as e: + print("Program reached the timeout set ({0} seconds). The command we executed was '{1}'".format(e.timeout, e.cmd)) + + +def clean_dir(*path): + dir = os.path.join(BASE_DIRECTORY, *path) + if os.path.exists(dir): + shutil.rmtree(dir) + os.mkdir(dir) + + +def set_working_directory(*path): + dir = os.path.join(BASE_DIRECTORY, *path) + os.chdir(dir) + + +def visualize(): + """ + Visualizes the benchmark results + """ + clean_dir("diagrams") + set_working_directory("reporting") + subprocess.call(["Rscript", "visualize.R", os.path.join(BASE_DIRECTORY, "config", "reporting.json")]) + + +def extract_results(): + """ + Extracts the benchmark results + """ + clean_dir("results") + set_working_directory("reporting") + subprocess.call(["Rscript", "check_results.R"]) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("-b", "--build", + help="build the project", + action="store_true") + parser.add_argument("-m", "--measure", + help="run the benchmark", + action="store_true") + parser.add_argument("-s", "--skip-tests", + help="skip JUNIT tests", + action="store_true") + parser.add_argument("-v", "--visualize", + help="create visualizations", + action="store_true") + parser.add_argument("-e", "--extract", + help="extract results", + action="store_true") + parser.add_argument("-t", "--test", + help="run test", + action="store_true") + parser.add_argument("-d", "--debug", + help="set debug to true", + action="store_true") + args = parser.parse_args() + + + set_working_directory("config") + with open("config.json", "r") as config_file: + config = json.load(config_file, object_hook = JSONObject) + + if args.debug: + os.environ['Debug'] = 'true' + if args.build: + build(config, args.skip_tests) + if args.measure: + benchmark(config) + if args.test: + build(config, False) + if args.visualize: + visualize() + if args.extract: + extract_results() + + # if there are no args, execute a full sequence + # with the test and the visualization/reporting + no_args = all(val==False for val in vars(args).values()) + if no_args: + build(config, False) + benchmark(config) + visualize() + extract_results()