Merge remote-tracking branch 'sle/artifact-evaluation'

trainbenchmark/trainbenchmark-alternate-scripts/build.gradle

+dependencies {
+	compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.2.3'
+	compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.3.0'
+
+	compile project(':trainbenchmark-generator')
+	compile project(':trainbenchmark-generator-emf')
+	compile project(':trainbenchmark-generator-graph-tinkerpop')
+	compile project(':trainbenchmark-generator-json4ag')
+	compile project(':trainbenchmark-generator-dot')
+	compile project(':trainbenchmark-tool')
+	compile project(':trainbenchmark-tool-jastadd-base')
+	compile project(':trainbenchmark-tool-jastadd-namelookup')
+	compile project(':trainbenchmark-tool-jastadd-namelookup-base')
+	compile project(':trainbenchmark-tool-jastadd-namelookup-incremental')
+	compile project(':trainbenchmark-tool-jastadd-optimized')
+	compile project(':trainbenchmark-tool-jastadd-optimized-base')
+	compile project(':trainbenchmark-tool-jastadd-optimized-incremental')
+//	compile project(':trainbenchmark-tool-jastadd-specialized')
+//	compile project(':trainbenchmark-tool-jastadd-specialized-base')
+//	compile project(':trainbenchmark-tool-jastadd-specialized-incremental')
+//	compile project(':trainbenchmark-tool-jastadd-relast')
+//	compile project(':trainbenchmark-tool-jastadd-relast-incremental')
+	compile project(':trainbenchmark-tool-tinkergraph')
+	compile project(':trainbenchmark-tool-viatra')
+}
+
+task cleanResults(type: Delete) {
+	doLast {
+		def resultDir = "../results"
+		file(resultDir).eachDir{ dir ->
+			delete "${dir}"
+		}
+	}
+}
+
+task combinedBenchmark(dependsOn: 'classes', type: JavaExec) {
+	group = 'Benchmark'
+	description = 'Runs the \'classic\' TrainBenchmark script'
+	main = 'de.tudresden.inf.st.train.scripts.BenchmarkMain'
+	classpath = sourceSets.main.runtimeClasspath
+	args 'combined'
+}
+
+task individualRepairBenchmark(dependsOn: 'classes', type: JavaExec) {
+	group = 'Benchmark'
+	main = 'de.tudresden.inf.st.train.scripts.BenchmarkMain'
+	classpath = sourceSets.main.runtimeClasspath
+	args 'individual_repair'
+}
+
+task individualInjectBenchmark(dependsOn: 'classes', type: JavaExec) {
+	group = 'Benchmark'
+	main = 'de.tudresden.inf.st.train.scripts.BenchmarkMain'
+	classpath = sourceSets.main.runtimeClasspath
+	args 'individual_inject'
+}
+
+task individualIncrementalBenchmark(dependsOn: 'classes', type: JavaExec) {
+	group = 'Benchmark'
+	main = 'de.tudresden.inf.st.train.scripts.BenchmarkMain'
+	classpath = sourceSets.main.runtimeClasspath
+	args 'individual_incremental'
+}
+
+
+task generate(dependsOn: 'classes', type: JavaExec) {
+	group = 'Benchmark'
+	description = 'Generates the models for all benchmarks'
+	main = 'de.tudresden.inf.st.train.scripts.GenerateMain'
+	classpath = sourceSets.main.runtimeClasspath
+}
+
+//task memory(dependsOn: 'classes', type: JavaExec) {
+//	group = 'Benchmark'
+//	main = 'MemoryScript'
+//	classpath = sourceSets.main.runtimeClasspath
+//}
+//
+//task qpt(dependsOn: 'classes', type: JavaExec) {
+//	group = 'Benchmark'
+//	main = 'QueryPlanTester'
+//	classpath = sourceSets.main.runtimeClasspath
+//}

trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkMain.java

+package de.tudresden.inf.st.train.scripts;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import hu.bme.mit.trainbenchmark.benchmark.config.*;
+import hu.bme.mit.trainbenchmark.benchmark.result.ResultHelper;
+import hu.bme.mit.trainbenchmark.benchmark.runcomponents.BenchmarkRunner;
+import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
+import hu.bme.mit.trainbenchmark.constants.RailwayOperation;
+
+import java.io.IOException;
+import java.util.*;
+
+/**
+ * Main entry point to start benchmarks.
+ * Created by rschoene on 10/12/17.
+ */
+public class BenchmarkMain {
+
+	@SuppressWarnings("unchecked")
+	private static void runBenchmarkSeries(BenchmarkConfigBaseBuilder configBaseBuilder, BenchmarkConfigBuilder configBuilder,
+	                                       ExecutionConfig ec, ModelSetConfig modelSetConfig, BenchmarkSettingsBasics bbs) {
+		Map<String, String> jvmSettingsMap = Utils.createJvmSettingsMap(bbs);
+		String jvmDefaultSettings = jvmSettingsMap.getOrDefault("*",
+				"-Xms{Xms} -Xmx{Xmx} -server -Xverify:none");
+		try {
+			for (int size = modelSetConfig.getMinSize(); size <= modelSetConfig.getMaxSize(); size *= 2) {
+				String modelFilename = "railway-" + modelSetConfig.getModelVariant() + "-" + size;
+
+				System.out.println("------------------------------------------------------------");
+				System.out.println("Model: " + modelFilename);
+				System.out.println("------------------------------------------------------------");
+
+				configBaseBuilder.setModelFilename(modelFilename);
+				BenchmarkConfigBase configBase = configBaseBuilder.createConfigBase();
+				BenchmarkConfig config = (BenchmarkConfig) configBuilder.setConfigBase(configBase).createConfig();
+
+				if (!bbs.isDryRun()) {
+					ResultHelper.prepare(config, ec);
+				}
+				String jvmSetting = jvmSettingsMap.getOrDefault(config.getToolName(), jvmDefaultSettings);
+
+				int exitValue = BenchmarkRunner.runPerformanceBenchmark(config, ec, jvmSetting);
+				if (exitValue != 0) {
+					System.out.println("Timeout or error occurred, skipping models for larger sizes. Error code: " + exitValue);
+					break;
+				}
+			}
+		} catch (InterruptedException | IOException e) {
+			System.out.println("Exception occurred during execution.");
+			e.printStackTrace();
+		}
+	}
+
+	public static void main(String args[]) throws Exception {
+		// set benchmark variant from program arguments
+		if (args.length == 0) {
+			System.err.println("Need to specify one argument identifying the benchmark variant");
+			System.exit(1);
+		}
+		String benchmarkVariant = args[0];
+
+		// read basic settings (min, maxsize, etc.)
+		ObjectMapper mapper = Utils.getMapper();
+		BenchmarkSettingsBasics bbs = Utils.readFromResource(mapper, "basic-settings.json",
+				BenchmarkSettingsBasics.class);
+		try {
+		BenchmarkSettingsBasics loca_bbs = Utils.readFromResource(mapper, "local-basic-settings.json",
+				BenchmarkSettingsBasics.class);
+			Utils.updateBasicsWithLocal(bbs, loca_bbs);
+		} catch (IOException e) {
+			System.out.println("No file 'local-basic-settings.json' found. Using default!");
+		}
+
+		// import tools to use
+		List<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> builders =
+				BenchmarkSettingsAvailableTools.getBuilders(bbs.getTools());
+		System.out.println(builders);
+
+		// read workloads based on benchmark variant
+		String workloadsFilename = "workloads.json";
+		BenchmarkSettingsVariantWorkloads bsvw = Utils.readFromResource(mapper, workloadsFilename,
+				BenchmarkSettingsVariantWorkloads.class);
+
+		Optional<BenchmarkSettingsWorkloads> workloadsOpt = bsvw.getVariants().stream()
+				.filter(w -> w.getName().equals(benchmarkVariant))
+				.findFirst();
+		if (!workloadsOpt.isPresent()) {
+			System.err.println("Could not find workloads for given variant, check " + workloadsFilename
+					+ " and argument " + benchmarkVariant);
+			System.exit(4);
+		}
+		BenchmarkSettingsWorkloads workloads = workloadsOpt.get();
+
+		// start benchmark
+		String now = ResultHelper.getNow();
+		ExecutionConfig ec = new ExecutionConfig(4000, 8000, bbs.isDryRun());
+
+		System.out.println("Please remember to stop all other Java processes.");
+		System.out.println();
+		System.out.println("If in doubt, check with this command:");
+		System.out.println("$ ps auxw | grep jav[a]");
+		System.out.println();
+		System.out.println("If there are other Java processes, use:");
+		System.out.println("$ killall -9 java");
+		System.out.println();
+		System.out.println("############################################################");
+		System.out.println("Benchmark parameters:");
+		System.out.println("- execution config: " + ec);
+		System.out.println(bbs);
+//		System.out.println("- range: minSize=" + bbs.getMinSize() +", maxSize=" + bbs.getMaxSize());
+//		System.out.println("- timeout: " + bbs.getTimeout());
+//		System.out.println("- runs: " + bbs.getRuns());
+		System.out.println("############################################################");
+		System.out.println();
+
+		workloads.getWorkloads().forEach(workloadConfiguration -> {
+			String workloadName = workloadConfiguration.getWorkloadName();
+			String modelVariant = workloadConfiguration.getModelVariant();
+			List<RailwayOperation> operations = workloadConfiguration.getOperations();
+			TransformationChangeSetStrategy strategy = workloadConfiguration.getStrategy();
+			int constant = workloadConfiguration.getConstant();
+			int queryTransformationCount = workloadConfiguration.getQueryTransformationCount();
+
+			System.out.println("============================================================");
+			System.out.println("Workload: " + workloadName);
+			System.out.println("============================================================");
+
+			ModelSetConfig modelSetConfig = new ModelSetConfig(modelVariant, bbs.getMinSize(), bbs.getMaxSize());
+			BenchmarkConfigBaseBuilder bcbb = new BenchmarkConfigBaseBuilder()
+					.setBenchmarkId(now).setTimeout(bbs.getTimeout()).setRuns(bbs.getRuns())
+					.setOperations(operations).setWorkload(workloadName)
+					.setQueryTransformationCount(queryTransformationCount).setTransformationConstant(constant)
+					.setTransformationChangeSetStrategy(strategy);
+
+			builders.forEach( bcb -> runBenchmarkSeries(bcbb, bcb, ec, modelSetConfig, bbs));
+		});
+
+//		if (binding.variables.get("reportUrl")) {
+//			BenchmarkReporter.reportReady(reportUrl)
+//		}
+	}
+
+}

trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableFormats.java

+package de.tudresden.inf.st.train.scripts;
+
+import de.tudresden.inf.st.trainbenchmark.generator.json4ag.config.Json4AgGeneratorConfigBuilder;
+import hu.bme.mit.trainbenchmark.generator.config.*;
+import hu.bme.mit.trainbenchmark.generator.emf.config.EmfGeneratorConfigBuilder;
+import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphFormat;
+import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphGeneratorConfigBuilder;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * Format settings of a benchmark shared throughout all variants.
+ * Created by rschoene on 10/13/17.
+ */
+public class BenchmarkSettingsAvailableFormats {
+	/**
+	 * Create a map, from projectName to a set of all builders in this project
+	 * @return a mapping from projectName to all builders
+	 */
+	public static Map<String, List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>>>
+	getAvailableFormats() {
+		// { projectName -> [builder] }
+		GeneratorConfigBase dummyConfigBase = new GeneratorConfigBaseBuilder()
+				.setScenario(Scenario.BATCH).setSize(1)
+				.createGeneratorConfigBase();
+		return getAllBuilders().stream().collect(Collectors.toMap(
+				builder -> builder.setConfigBase(dummyConfigBase).createConfig().getProjectName(),
+				s -> new ArrayList<>(Collections.singletonList(s)),
+				(oldValue, newValue) -> { oldValue.addAll(newValue); return oldValue; }));
+	}
+
+	private static GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>
+	getBuilderByName(String className) {
+		Optional<Object> result = Utils.maybeCreateNewInstance(className);
+		return (GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>) result.orElse(null);
+	}
+
+	private static List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>>
+	getAllBuilders() {
+		return Arrays.asList(
+				new EmfGeneratorConfigBuilder(),
+				new TinkerGraphGeneratorConfigBuilder().setGraphFormat(TinkerGraphFormat.GRAPHML),
+				new Json4AgGeneratorConfigBuilder()
+		);
+	}
+
+	/**
+	 * Get all builders with the given names of formats. The format name in the config every builder creates
+	 * must match one of the given project names.
+	 * @param formatNames names of formats
+	 * @return a set of matching generator config builders
+	 */
+	public static List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>>
+	getBuilders(String ... formatNames) {
+		Map<String, List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>>>
+				availableFormats = getAvailableFormats();
+		return Arrays.stream(formatNames).map(name -> {
+			List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>>
+					builders = availableFormats.get(name);
+			if (builders.isEmpty()) {
+				throw new IllegalArgumentException("Could not find format " + name + ", available formats are: "
+						+ availableFormats.keySet());
+			}
+			return builders;
+		}).reduce((set1, set2) -> {
+			set1.addAll(set2);
+			return set1;
+		}).orElse(Collections.emptyList());
+	}
+}

trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableTools.java

+package de.tudresden.inf.st.train.scripts;
+
+import de.tudresden.inf.st.train.jastadd.config.JastaddNameLookupBenchmarkConfigBuilder;
+import de.tudresden.inf.st.train.jastadd.config.JastaddNameLookupIncrementalBenchmarkConfigBuilder;
+import de.tudresden.inf.st.train.jastadd.config.JastaddOptimizedBenchmarkConfigBuilder;
+import de.tudresden.inf.st.train.jastadd.config.JastaddOptimizedIncrementalBenchmarkConfigBuilder;
+import hu.bme.mit.trainbenchmark.benchmark.config.*;
+import hu.bme.mit.trainbenchmark.benchmark.tinkergraph.config.TinkerGraphBenchmarkConfigBuilder;
+import hu.bme.mit.trainbenchmark.benchmark.viatra.config.ViatraBackend;
+import hu.bme.mit.trainbenchmark.benchmark.viatra.config.ViatraBenchmarkConfigBuilder;
+
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Tool settings of a benchmark shared throughout all variants
+ * Created by rschoene on 10/12/17.
+ */
+public class BenchmarkSettingsAvailableTools {
+	public static Map<String, BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>>
+	getAvailableTools() {
+		// { toolName -> builder }
+		BenchmarkConfigBase dummyConfigBase = new BenchmarkConfigBaseBuilder()
+				.setBenchmarkId("").setTimeout(0L).setRuns(0).setModelFilename("")
+				.setOperations(new ArrayList<>()).setWorkload("")
+				.setTransformationChangeSetStrategy(TransformationChangeSetStrategy.FIXED)
+				.setQueryTransformationCount(0).setTransformationConstant(0)
+				.createConfigBase();
+		return getAllBuilders().stream().collect(Collectors.toMap(
+				builder -> builder.setConfigBase(dummyConfigBase).createConfig().getToolName(),
+				Function.identity()));
+	}
+
+	private static BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>
+	getBuilderByName(String className) {
+		Optional<Object> result = Utils.maybeCreateNewInstance(className);
+		return (BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>) result.orElse(null);
+	}
+
+	private static List<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>>
+	getAllBuilders() {
+		return Arrays.asList(
+				new JastaddNameLookupBenchmarkConfigBuilder(),
+				new JastaddNameLookupIncrementalBenchmarkConfigBuilder(),
+				new JastaddOptimizedBenchmarkConfigBuilder(),
+				new JastaddOptimizedIncrementalBenchmarkConfigBuilder(),
+				new TinkerGraphBenchmarkConfigBuilder(),
+				new ViatraBenchmarkConfigBuilder().setBackend(ViatraBackend.INCREMENTAL),
+				new ViatraBenchmarkConfigBuilder().setBackend(ViatraBackend.LOCAL_SEARCH)
+		);
+	}
+
+	/**
+	 * Get all builders with the given names of tools. The tool name in the config every builder creates must match one
+	 * of the given tool names.
+	 * @param toolNames names of tools
+	 * @return a list of matching benchmark config builders
+	 */
+	public static List<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>>
+	getBuilders(String ... toolNames) {
+		Map<String, BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> availableTools = getAvailableTools();
+		return Arrays.stream(toolNames).map(name -> {
+			BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>> builder = availableTools.get(name);
+			if (builder == null) {
+				throw new IllegalArgumentException("Could not find tool " + name + ", available tools are: " + availableTools.keySet());
+			}
+			return builder;
+		}).collect(Collectors.toList());
+	}
+}

trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsBasics.java

+package de.tudresden.inf.st.train.scripts;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonSetter;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Basic settings bean of a benchmark shared throughout all variants.
+ * Created by rschoene on 10/12/17.
+ */
+public class BenchmarkSettingsBasics {
+
+	static class JVMSetting {
+		private List<String> applyTo;
+		private String args;
+
+		public List<String> getApplyTo() {
+			return applyTo;
+		}
+
+		public void setApplyTo(List<String> applyTo) {
+			this.applyTo = applyTo;
+		}
+
+		public String getArgs() {
+			return args;
+		}
+
+		public void setArgs(String args) {
+			this.args = args;
+		}
+	}
+	@JsonInclude(JsonInclude.Include.NON_DEFAULT)
+	private Integer minSize = null;
+	@JsonInclude(JsonInclude.Include.NON_DEFAULT)
+	private Integer maxSize = null;
+	@JsonInclude(JsonInclude.Include.NON_DEFAULT)
+	private Long timeout = null;
+	@JsonInclude(JsonInclude.Include.NON_DEFAULT)
+	private Integer runs = null;
+	@JsonInclude(JsonInclude.Include.NON_DEFAULT)
+	private Boolean dryRun = null;
+	private String[] tools;
+	private String[] formats;
+	private List<JVMSetting> jvmSettings;
+
+	/** Minimum model size */
+	public Integer getMinSize() {
+		return minSize;
+	}
+
+	public void setMinSize(int minSize) {
+		this.minSize = minSize;
+	}
+
+	/** Maximum model size */
+	public Integer getMaxSize() {
+		return maxSize;
+	}
+
+	public void setMaxSize(int maxSize) {
+		this.maxSize = maxSize;
+	}
+
+	/** Timeout in seconds for each single run. */
+	public Long getTimeout() {
+		return timeout;
+	}
+
+	public void setTimeout(long timeout) {
+		this.timeout = timeout;
+	}
+
+	/** Number of runs per tool-workload combination. */
+	public Integer getRuns() {
+		return runs;
+	}
+
+	public void setRuns(int runs) {
+		this.runs = runs;
+	}
+
+	/** Don't run any Java processes. */
+	public Boolean isDryRun() {
+		return dryRun;
+	}
+
+	@JsonSetter("dry-run")
+	public void setDryRun(boolean dryRun) {
+		this.dryRun = dryRun;
+	}
+
+	/** List of tool names to use */
+	public String[] getTools() {
+		return tools;
+	}
+
+	public void setTools(String[] tools) {
+		this.tools = tools;
+	}
+
+	/** List of format names to use */
+	public String[] getFormats() {
+		return formats;
+	}
+
+	public void setFormats(String[] formats) {
+		this.formats = formats;
+	}
+
+	public List<JVMSetting> getJvmSettings() {
+		return jvmSettings;
+	}
+
+	public void setJvmSettings(List<JVMSetting> jvmSettings) {
+		this.jvmSettings = jvmSettings;
+	}
+
+	@Override
+	public String toString() {
+		return "BenchmarkSettingsBasics{" +
+				"minSize=" + minSize +
+				", maxSize=" + maxSize +
+				", timeout=" + timeout +
+				", runs=" + runs +
+				", dryRun=" + dryRun +
+				", tools=" + Arrays.toString(tools) +
+				'}';
+	}
+}

trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsVariantWorkloads.java

+package de.tudresden.inf.st.train.scripts;
+
+import java.util.List;
+
+/**
+ * List of all workload settings by variant.
+ * Created by rschoene on 10/12/17.
+ */
+public class BenchmarkSettingsVariantWorkloads {
+	private List<BenchmarkSettingsWorkloads> variants;
+
+	public List<BenchmarkSettingsWorkloads> getVariants() {
+		return variants;
+	}
+
+	public void setVariants(List<BenchmarkSettingsWorkloads> variants) {
+		this.variants = variants;
+	}
+}

trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsWorkload.java

+package de.tudresden.inf.st.train.scripts;
+
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import hu.bme.mit.trainbenchmark.benchmark.config.TransformationChangeSetStrategy;
+import hu.bme.mit.trainbenchmark.constants.RailwayOperation;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Benchmark setting workload bean.
+ * Created by rschoene on 10/12/17.
+ */
+public class BenchmarkSettingsWorkload {
+	private String workloadName;
+	private String modelVariant;
+	private List<RailwayOperation> operations;
+	private TransformationChangeSetStrategy strategy;
+//	private String strategy;
+	private int constant;
+	private int queryTransformationCount;
+	@JsonIgnore
+	private Map<String, RailwayOperation> railwayOps;
+
+	public BenchmarkSettingsWorkload() {
+		railwayOps = Arrays.stream(RailwayOperation.values()).collect(Collectors.toMap(
+				RailwayOperation::toString, Function.identity()
+		));
+	}
+
+	public String getWorkloadName() {
+		return workloadName;
+	}
+
+	public void setWorkloadName(String workloadName) {
+		this.workloadName = workloadName;
+	}
+
+	public String getModelVariant() {
+		return modelVariant;
+	}
+
+	public void setModelVariant(String modelVariant) {
+		this.modelVariant = modelVariant;
+	}
+
+	public List<RailwayOperation> getOperations() {
+		return operations;
+	}
+
+	public void setOperations(List<String> operations) {
+		this.operations = operations.stream().map(op -> railwayOps.get(op)).collect(Collectors.toList());
+	}
+
+	public TransformationChangeSetStrategy getStrategy() {
+		return strategy;
+	}
+
+	public void setStrategy(TransformationChangeSetStrategy strategy) {
+		this.strategy = strategy;
+	}
+
+	public int getConstant() {
+		return constant;
+	}
+
+	public void setConstant(int constant) {
+		this.constant = constant;
+	}
+
+	public int getQueryTransformationCount() {
+		return queryTransformationCount;
+	}
+
+	public void setQueryTransformationCount(int queryTransformationCount) {
+		this.queryTransformationCount = queryTransformationCount;
+	}
+}

trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsWorkloads.java

+package de.tudresden.inf.st.train.scripts;
+
+import java.util.List;
+
+/**
+ * Workload settings bean.
+ * Created by rschoene on 10/12/17.
+ */
+public class BenchmarkSettingsWorkloads
+{
+	private String name;
+	private List<BenchmarkSettingsWorkload> workloads;
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(String name) {
+		this.name = name;
+	}
+
+	public List<BenchmarkSettingsWorkload> getWorkloads() {
+		return workloads;
+	}
+
+	public void setWorkloads(List<BenchmarkSettingsWorkload> workloads) {
+		this.workloads = workloads;
+	}
+}

trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkVariant.java

+package de.tudresden.inf.st.train.scripts;
+
+/**
+ * Possible variants to run a benchmark in.
+ * Created by rschoene on 10/12/17.
+ */
+public enum BenchmarkVariant {
+	COMBINED,
+	INDIVIDUAL_INJECT,
+	INDIVIDUAL_REPAIR,
+	INDIVIDUAL_INCREMENTAL
+}

trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/GenerateMain.java

+package de.tudresden.inf.st.train.scripts;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
+import hu.bme.mit.trainbenchmark.generator.config.*;
+import hu.bme.mit.trainbenchmark.generator.runner.GeneratorRunner;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * Main entry point to generate models.
+ * Created by rschoene on 10/13/17.
+ */
+public class GenerateMain {
+	public static void main(String[] args) throws Exception {
+		// read basic settings (min, maxsize, etc.)
+		ObjectMapper mapper = Utils.getMapper();
+		BenchmarkSettingsBasics bbs = Utils.readFromResource(mapper, "basic-settings.json",
+				BenchmarkSettingsBasics.class);
+		try {
+			BenchmarkSettingsBasics loca_bbs = Utils.readFromResource(mapper, "local-basic-settings.json",
+					BenchmarkSettingsBasics.class);
+			Utils.updateBasicsWithLocal(bbs, loca_bbs);
+		} catch (IOException e) {
+			System.out.println("No file 'local-basic-settings.json' found. Using default!");
+		}
+
+		// import formats to use
+		List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>> builders =
+				BenchmarkSettingsAvailableFormats.getBuilders(bbs.getFormats());
+		System.out.println(builders);
+		ExecutionConfig ec = new ExecutionConfig(4000, 8000, bbs.isDryRun());
+
+		// start generating (exclude Scenario.MINIMAL)
+		for (Scenario scenario : new Scenario[]{Scenario.BATCH, Scenario.INJECT, Scenario.REPAIR}) {
+			builders.forEach( generatorConfigBuilder -> {
+				try {
+					for (int size = bbs.getMinSize(); size <= bbs.getMaxSize(); size *= 2) {
+						System.out.println("Scenario: " + scenario + ", size: " + size);
+
+						GeneratorConfigBase configBase = new GeneratorConfigBaseBuilder()
+								.setSize(size).setScenario(scenario)
+								.createGeneratorConfigBase();
+						GeneratorConfig config = generatorConfigBuilder.setConfigBase(configBase).createConfig();
+
+						int exitValue = GeneratorRunner.run(config, ec);
+						if (exitValue != 0) {
+							System.out.println("Timeout or error occurred, skipping models for larger sizes. Error code: "
+									+ exitValue);
+							break;
+						}
+					}
+				} catch (Exception e) {
+					System.out.println("Exception occurred during execution.");
+				}
+			});
+		}
+	}
+
+}
+
+/*
+import de.tudresden.inf.st.trainbenchmark.generator.json4ag.config.Json4AgGeneratorConfigBuilder
+import de.tudresden.inf.st.trainbenchmark.generator.json4agref.config.Json4AgRefGeneratorConfigBuilder
+import hu.bme.mit.trainbenchmark.config.ExecutionConfig
+import hu.bme.mit.trainbenchmark.generator.config.GeneratorConfigBase
+import hu.bme.mit.trainbenchmark.generator.config.Scenario
+import hu.bme.mit.trainbenchmark.generator.emf.config.EmfGeneratorConfigBuilder
+import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphFormat
+import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphGeneratorConfigBuilder
+import hu.bme.mit.trainbenchmark.generator.runner.GeneratorRunner
+
+def ec = new ExecutionConfig(4000, 6000)
+def minSize = 1
+/* def maxSize = 2048 *
+def maxSize = 256
+
+	def scenarios = [
+		Scenario.BATCH,
+		Scenario.INJECT,
+		Scenario.REPAIR,
+		]
+
+		def formats = [
+		new EmfGeneratorConfigBuilder(),
+		new TinkerGraphGeneratorConfigBuilder().setGraphFormat(TinkerGraphFormat.GRAPHML),
+		new Json4AgGeneratorConfigBuilder(),
+		new Json4AgRefGeneratorConfigBuilder()
+		]
+
+		for (scenario in scenarios) {
+		formats.each { generatorConfigBuilder ->
+		try {
+		for (def size = minSize; size <= maxSize; size *= 2) {
+		println("Scenario: ${scenario}, size: ${size}")
+
+		def configBase = new GeneratorConfigBase(scenario, size)
+		def config = generatorConfigBuilder.setConfigBase(configBase).createConfig()
+
+		def exitValue = GeneratorRunner.run(config, ec)
+		if (exitValue != 0) {
+		println "Timeout or error occured, skipping models for larger sizes. Error code: ${exitValue}"
+		break
+		}
+		}
+		} catch (all) {
+		println "Exception occured during execution."
+		}
+		}
+		}
+
+ */

trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/Utils.java

+package de.tudresden.inf.st.train.scripts;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * Utility methods for the main entry points.
+ * Created by rschoene on 10/13/17.
+ */
+public class Utils {
+	static Map<String, String> createJvmSettingsMap(BenchmarkSettingsBasics bbs) {
+		Map<String, String> result = new HashMap<>();
+		for (BenchmarkSettingsBasics.JVMSetting setting : bbs.getJvmSettings()) {
+			for (String toolName : setting.getApplyTo()) {
+				result.put(toolName, setting.getArgs());
+			}
+		}
+		return result;
+	}
+
+	private static File readFromResource(String filename) throws IOException {
+		URL basicSettingsURL = BenchmarkMain.class.getClassLoader().getResource(filename);
+		if (basicSettingsURL == null) {
+			System.err.println();
+			throw new IOException("Could not access " + filename + ". Exiting.");
+		}
+		return new File(basicSettingsURL.getFile());
+	}
+
+	static <T> T readFromResource(ObjectMapper mapper, String filename, Class<T> clazz) throws IOException {
+		File basicSettingsFile = readFromResource(filename);
+		T result = null;
+		try {
+			result = mapper.readValue(basicSettingsFile, clazz);
+		} catch (Exception e) {
+			System.err.println("Could not load '" + filename + "'. Exiting.");
+			e.printStackTrace();
+			System.exit(2);
+		}
+		return result;
+	}
+
+	static void updateBasicsWithLocal(BenchmarkSettingsBasics base, BenchmarkSettingsBasics local)
+			throws IllegalStateException {
+		List<Method> setter = Arrays.stream(BenchmarkSettingsBasics.class.getMethods())
+				.filter(m -> m.getName().startsWith("set"))
+				.collect(Collectors.toList());
+		for (Method method : BenchmarkSettingsBasics.class.getMethods()) {
+			final String name = method.getName().startsWith("get") ?
+					method.getName().substring(3) :
+					( method.getName().startsWith("is") ? method.getName().substring(2) : null);
+			if (name == null) {
+				continue;
+			}
+			try {
+				Object result = method.invoke(local);
+				if (result != null) {
+					Optional<Method> matchingSetter = setter.stream()
+							.filter(m -> m.getName().endsWith(name))
+							.findFirst();
+					if (matchingSetter.isPresent()) {
+						matchingSetter.get().invoke(base, result);
+					}
+				}
+				// now do sanity check if set in updated base
+				result = method.invoke(base);
+				if (result == null) {
+					throw new IllegalStateException("Basic setting for " + name + " is not given.");
+				}
+			} catch (IllegalAccessException | InvocationTargetException e) {
+				e.printStackTrace();
+			}
+		}
+	}
+
+	static ObjectMapper getMapper() {
+		ObjectMapper mapper = new ObjectMapper();
+		mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
+		return mapper;
+	}
+
+	static Optional<Object> maybeCreateNewInstance(final String className) {
+		try {
+			return Optional.of(Class.forName(className).newInstance());
+		} catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
+			System.err.println("Could not find builder class '" + className + "'");
+			e.printStackTrace();
+			return Optional.empty();
+		}
+	}
+}

trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/.gitignore

+# basic settings can still be commited using --force
+local-basic-settings.json

trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/basic-settings.json

+{
+  "minSize": 1,
+  "maxSize": 8,
+  "timeout": 900,
+  "runs": 1,
+  "dry-run": true,
+  "tools": [
+    "Jastadd (Name Lookup)",
+    "Jastadd (Incremental Name Lookup)",
+    "Jastadd (Optimized)",
+    "Jastadd (Optimized Incremental)",
+    "TinkerGraph",
+    "VIATRA (Incremental)",
+    "VIATRA (Local Search)"
+  ],
+  "formats": [
+    "emf",
+    "json4ag",
+    "graph-tinkerpop"
+  ],
+  "jvmSettings": [
+    {
+      "applyTo": ["*"],
+      "args": "-Xms%(Xms) -Xmx%(Xmx) -server -Xverify:none"
+//      "args": "-Xms%(Xms) -Xmx%(Xmx) -server -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:StartFlightRecording=dumponexit=true -Xverify:none"
+    },
+    {
+      "applyTo": ["Racr (CPP)", "Racr (Python)", "Racr (Scheme)"],
+      "args": "-Xms50M -Xmx50M -server -Xverify:none"
+    }
+  ]
+}

trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/workloads.json

+{
+  "variants": [
+    {
+      "name": "individual_repair",
+      "workloads": [
+        {
+          "workloadName": "ConnectedSegments",
+          "operations": [
+            "ConnectedSegmentsRepair"
+          ],
+          "constant": 5,
+          "queryTransformationCount": 8,
+          "modelVariant": "repair",
+          "strategy": "PROPORTIONAL"
+        },
+        {
+          "workloadName": "PosLength",
+          "operations": [
+            "PosLengthRepair"
+          ],
+          "constant": 5,
+          "queryTransformationCount": 8,
+          "modelVariant": "repair",
+          "strategy": "PROPORTIONAL"
+        },
+        {
+          "workloadName": "RouteSensor",
+          "operations": [
+            "RouteSensorRepair"
+          ],
+          "constant": 5,
+          "queryTransformationCount": 8,
+          "modelVariant": "repair",
+          "strategy": "PROPORTIONAL"
+        },
+        {
+          "workloadName": "SemaphoreNeighbor",
+          "operations": [
+            "SemaphoreNeighborRepair"
+          ],
+          "constant": 5,
+          "queryTransformationCount": 8,
+          "modelVariant": "repair",
+          "strategy": "PROPORTIONAL"
+        },
+        {
+          "workloadName": "SwitchMonitored",
+          "operations": [
+            "SwitchMonitoredRepair"
+          ],
+          "constant": 5,
+          "queryTransformationCount": 8,
+          "modelVariant": "repair",
+          "strategy": "PROPORTIONAL"
+        },
+        {
+          "workloadName": "SwitchSet",
+          "operations": [
+            "SwitchSetRepair"
+          ],
+          "constant": 5,
+          "queryTransformationCount": 8,
+          "modelVariant": "repair",
+          "strategy": "PROPORTIONAL"
+        }
+      ]
+    },
+    {
+      "name": "combined",
+      "workloads": [
+        {
+          "workloadName": "Repair",
+          "operations": [
+            "ConnectedSegmentsRepair",
+            "PosLengthRepair",
+            "RouteSensorRepair",
+            "SemaphoreNeighborRepair",
+            "SwitchMonitoredRepair",
+            "SwitchSetRepair"
+          ],
+          "constant": 5,
+          "queryTransformationCount": 8,
+          "modelVariant": "repair",
+          "strategy": "PROPORTIONAL"
+        },
+        {
+          "workloadName": "Inject",
+          "operations": [
+            "ConnectedSegments",
+            "PosLength",
+            "RouteSensor",
+            "SemaphoreNeighbor",
+            "SwitchMonitored",
+            "SwitchSet",
+            "ConnectedSegmentsInject",
+            "PosLengthInject",
+            "RouteSensorInject",
+            "SemaphoreNeighborInject",
+            "SwitchMonitoredInject",
+            "SwitchSetInject"
+          ],
+          "constant": 10,
+          "queryTransformationCount": 12,
+          "modelVariant": "inject",
+          "strategy": "FIXED"
+        }
+      ]
+    },
+    {
+      "name": "individual_inject",
+      "workloads": [
+        {
+          "workloadName": "ConnectedSegments",
+          "operations": [
+            "ConnectedSegments",
+            "ConnectedSegmentsInject"
+          ],
+          "constant": 10,
+          "queryTransformationCount": 12,
+          "modelVariant": "inject",
+          "strategy": "FIXED"
+        },
+        {
+          "workloadName": "PosLength",
+          "operations": [
+            "PosLength",
+            "PosLengthInject"
+          ],
+          "constant": 10,
+          "queryTransformationCount": 12,
+          "modelVariant": "inject",
+          "strategy": "FIXED"
+        },
+        {
+          "workloadName": "RouteSensor",
+          "operations": [
+            "RouteSensor",
+            "RouteSensorInject"
+          ],
+          "constant": 10,
+          "queryTransformationCount": 12,
+          "modelVariant": "inject",
+          "strategy": "FIXED"
+        },
+        {
+          "workloadName": "SemaphoreNeighbor",
+          "operations": [
+            "SemaphoreNeighbor",
+            "SemaphoreNeighborInject"
+          ],
+          "constant": 10,
+          "queryTransformationCount": 12,
+          "modelVariant": "inject",
+          "strategy": "FIXED"
+        },
+        {
+          "workloadName": "SwitchMonitored",
+          "operations": [
+            "SwitchMonitored",
+            "SwitchMonitoredInject"
+          ],
+          "constant": 10,
+          "queryTransformationCount": 12,
+          "modelVariant": "inject",
+          "strategy": "FIXED"
+        },
+        {
+          "workloadName": "SwitchSet",
+          "operations": [
+            "SwitchSet",
+            "SwitchSetInject"
+          ],
+          "constant": 10,
+          "queryTransformationCount": 12,
+          "modelVariant": "inject",
+          "strategy": "FIXED"
+        }
+      ]
+    },
+    {
+      "name": "individual_incremental",
+      "workloads": [
+        {
+          "workloadName": "ConnectedSegments",
+          "operations": [
+            "ConnectedSegments",
+            "ConnectedSegmentsInject",
+            "ConnectedSegmentsRepair"
+          ],
+          "constant": 10,
+          "queryTransformationCount": 50,
+          "modelVariant": "inject",
+          "strategy": "FIXED"
+        },
+        {
+          "workloadName": "PosLength",
+          "operations": [
+            "PosLength",
+            "PosLengthInject",
+            "PosLengthRepair"
+          ],
+          "constant": 10,
+          "queryTransformationCount": 50,
+          "modelVariant": "inject",
+          "strategy": "FIXED"
+        },
+        {
+          "workloadName": "RouteSensor",
+          "operations": [
+            "RouteSensor",
+            "RouteSensorInject",
+            "RouteSensorRepair"
+          ],
+          "constant": 10,
+          "queryTransformationCount": 50,
+          "modelVariant": "inject",
+          "strategy": "FIXED"
+        },
+        {
+          "workloadName": "SemaphoreNeighbor",
+          "operations": [
+            "SemaphoreNeighbor",
+            "SemaphoreNeighborInject",
+            "SemaphoreNeighborRepair"
+          ],
+          "constant": 10,
+          "queryTransformationCount": 50,
+          "modelVariant": "inject",
+          "strategy": "FIXED"
+        },
+        {
+          "workloadName": "SwitchMonitored",
+          "operations": [
+            "SwitchMonitored",
+            "SwitchMonitoredInject",
+            "SwitchMonitoredRepair"
+          ],
+          "constant": 10,
+          "queryTransformationCount": 50,
+          "modelVariant": "inject",
+          "strategy": "FIXED"
+        },
+        {
+          "workloadName": "SwitchSet",
+          "operations": [
+            "SwitchSet",
+            "SwitchSetInject",
+            "SwitchSetRepair"
+          ],
+          "constant": 10,
+          "queryTransformationCount": 50,
+          "modelVariant": "inject",
+          "strategy": "FIXED"
+        }
+      ]
+    }
+  ]
+}

trainbenchmark/trainbenchmark-config/src/main/java/hu/bme/mit/trainbenchmark/config/ExecutionConfig.java

@@ -4,10 +4,16 @@ public class ExecutionConfig {
 
 	protected Integer initialMemory;
 	protected Integer maxMemory;
+	protected Boolean dryRun;
 	
-	public ExecutionConfig(final Integer initialMemory, final Integer maxMemory) {
+	public ExecutionConfig(final Integer initialMemory, final Integer maxMemory, final boolean dryRun) {
 		this.initialMemory = initialMemory;
 		this.maxMemory = maxMemory;
+		this.dryRun = dryRun;
+	}
+
+	public ExecutionConfig(final Integer initialMemory, final Integer maxMemory) {
+		this(initialMemory, maxMemory, true);
 	}
 
 	/**
@@ -42,6 +48,14 @@ public class ExecutionConfig {
 		return maxMemory + "M";
 	}
 
+	/**
+	 *
+	 * @return whether this benchmark run must not start any Java processes
+	 */
+	public Boolean getDryRun() {
+		return dryRun;
+	}
+
 	public static ExecutionConfig defaultExecutionConfig() {
 		return new ExecutionConfig(1000, 1000);
 	}

trainbenchmark/trainbenchmark-reporting/.gitignore

 Rplots.pdf
-
+local-merge_results.json

trainbenchmark/trainbenchmark-reporting/build.gradle

+task doMerge(type: Exec) {
+    group = 'Benchmark'
+    description = 'Merges the results'
+    commandLine './do-merge.sh'
+}
+
 task plot(type: Exec) {
+    group = 'Benchmark'
+    description = 'Plots the \'classic\' TrainBenchmark result'
     commandLine 'Rscript', 'report.R'
+    dependsOn doMerge
 }
 
 task plotIndividual(type: Exec) {
+    group = 'Benchmark'
+    description = 'Plots the individual TrainBenchmark results'
     commandLine 'Rscript', 'individual.R'
+    dependsOn doMerge
+}
+
+task plotToolwise(type: Exec) {
+    group = 'Benchmark'
+    description = 'Plots the individual TrainBenchmark results per tool'
+    commandLine './toolwise.sh'
+    dependsOn doMerge
 }

trainbenchmark/trainbenchmark-reporting/do-merge.sh

+python merge_results.py --result-dir ../results/ --create-run-dirs --create-toolwise-dirs $@

trainbenchmark/trainbenchmark-reporting/merge_results.json

+{
+  "tools": [
+    "tinkergraph",
+    "drools",
+    "mysql",
+    "kiama",
+    "jastadd-java-references",
+    "jastadd-java-references-incremental",
+    "jastadd-symbolic-references-incremental",
+    "jastadd-symbolic-references",
+    "sqlite",
+    "viatra",
+    "racr-cpp",
+    "racr-python",
+    "racr-scheme",
+    "neo4j",
+    "sesame",
+    "emfapi",
+    "rdf4j",
+    "epsilon",
+    "eclipseocl"
+  ],
+  "ignored": [
+  ],
+  "toolwise": [
+  ]
+}

trainbenchmark/trainbenchmark-reporting/merge_results.py

+#!/usr/bin/env python
+import argparse
+import csv
+import glob
+import json
+import logging
+import os
+import os.path
+import re
+import sys
+
+
+FORMAT = '%(asctime)s %(levelname)-8s %(threadName)-10s (%(filename)s:%(lineno)d): %(message)s'
+BENCHMARK_PATTERN = re.compile('.*-(BatchModel|Repair|Inject)Test.*')
+logger = logging.getLogger('merge_result')
+SIZE_PATTERN = re.compile('.*-railway-[^\\-]*-([^\\-]*)-.csv')
+NAME_PATTERN = re.compile('(times|matches)-([^\\-]*)-.*.csv')
+RUN_PATTERN = re.compile('run-(....)-(..)-(..)-(..)-(..)-(..)')
+RUN_REPLACMENT = r'\1_\2_\3 \4:\5:\6'
+
+
+def include_file_config(args):
+    def override_if_defined(key, convert=lambda x: x):
+        keyArgs = key.replace('-', '_')
+        value = content.get(key) or content.get(keyArgs)
+        if value:
+            setattr(args, keyArgs, convert(value))
+
+    # load config file
+    with open(args.file_config) as fdr:
+        content = json.load(fdr)
+    # update with local version, if existing
+    directory, basename = os.path.split(os.path.abspath(args.file_config))
+    local_config_file = os.path.join(directory, 'local-' + basename)
+    if os.path.exists(local_config_file):
+        with open(local_config_file) as fdr:
+            content.update(json.load(fdr))
+    else:
+        logger.debug('No local config file found.')
+    if not content.get('tools'):
+        logger.error('Key "tools" not found in config file "' + args.file_config + '". Exiting.')
+        sys.exit(1)
+    args.tools = content['tools']
+    override_if_defined('max-size', int)
+    override_if_defined('dry-run', bool)
+    override_if_defined('result-dir')
+    override_if_defined('create-run-dirs', bool)
+    override_if_defined('create-toolwise-dirs', bool)
+    override_if_defined('no-clean', bool)
+    override_if_defined('verbose', bool)
+
+
+def create_link(fileToLink, linkName, dry_run):
+    if dry_run:
+        return
+    if os.path.lexists(linkName):
+        os.unlink(linkName)
+    (logger.info if args.dry_run else logger.debug)('Linking %s to %s', fileToLink, linkName)
+    os.symlink(fileToLink, linkName)
+
+
+def ensure_directory(dir_name, dry_run):
+    if dry_run:
+        return
+    if not os.path.exists(dir_name):
+        logger.info('Creating %s', dir_name)
+        os.mkdir(dir_name)
+
+
+def exceeds(filename, max_size):
+    match = SIZE_PATTERN.match(filename)
+    return int(match.group(1)) > max_size if match else False
+
+
+def remove_if_there(the_list, element_to_remove):
+    if element_to_remove in the_list:
+        the_list.remove(element_to_remove)
+
+
+def new_run_name(old_run_name):
+    return RUN_PATTERN.sub(RUN_REPLACMENT, old_run_name)
+
+
+def copy_replace(fileTocopy, all_runs_dir, tool_name, run, dry_run):
+    """
+    Take fileTocopy, copy it to all_runs_dir, while replacing tool_name with run
+    in both its name and its content
+    """
+    run_name = new_run_name(run)
+    targetFile = os.path.join(
+        all_runs_dir, os.path.basename(fileTocopy).replace(tool_name, run_name))
+    first = True
+    with open(fileTocopy) as fdr_source, open(targetFile, 'w') as fdr_target:
+        for line in fdr_source:
+            if first:
+                first = False
+            else:
+                line = line.replace(tool_name, run_name)
+            fdr_target.write(line)
+
+
+def main(args):
+    """
+    Main process.
+
+    Used directory structure/variables:
+    results/                        -> result_dir
+        tools/                      -
+            tool1/                  -> tool_dir
+                tool1-run1/         -> run_dir
+                    times.csv       -
+                    matches.csv     -
+                run-list.csv        -
+                all-runs/           -> all_runs_dir
+                    times-run1.csv@ -
+        run1/                       -> global_run_dir
+        merged/                     -> merged_dir
+            individual/             -> merged_dir_individual
+                times.csv@          -
+            combined/               -> merged_dir_benchmark
+                times.csv@          -
+    """
+    log_action = logger.info if args.dry_run else logger.debug
+
+    # Gathering paths, creating top-level directories
+    result_dir = os.path.abspath(args.result_dir)
+    merged_dir = os.path.join(result_dir, 'merged')
+    merged_dir_benchmark = os.path.join(merged_dir, 'benchmark')
+    merged_dir_individual = os.path.join(merged_dir, 'individual')
+    for dir_name in (merged_dir, merged_dir_benchmark, merged_dir_individual):
+        ensure_directory(dir_name, args.dry_run)
+
+    # Gathering tools
+    tools = []
+    reader = csv.reader(args.tools)
+    next(reader)
+    for row in reader:
+        if not row:
+            continue
+        tools.append(row[0])
+    logger.debug('result_dir: %s, tools: %s', result_dir, tools)
+
+    # Clean symlinks if requested or max_size is set
+    if (args.clean or args.max_size) and not args.dry_run:
+        for dir_to_clean in [merged_dir, merged_dir_benchmark, merged_dir_individual]:
+            for link in os.listdir(dir_to_clean):
+                linkName = os.path.join(dir_to_clean, link)
+                if os.path.islink(linkName):
+                    os.unlink(linkName)
+    if (args.clean or args.create_toolwise_dirs) and not args.dry_run:
+        for linkName in glob.iglob(os.path.join(result_dir, 'tools', '*', 'all-runs', '*.csv')):
+            os.remove(linkName)
+
+    # Merge results
+    for tool in tools:
+        if tool.startswith('#'):
+            logger.debug('Ignoring tool "%s"', tool[1:])
+            continue
+        already_merged = []
+        tool_dir = os.path.join(result_dir, 'tools', tool)
+        if not os.path.exists(tool_dir):
+            logger.warn('Tool not found: %s', tool)
+            continue
+        all_runs_dir = os.path.join(tool_dir, 'all-runs')
+        ensure_directory(all_runs_dir, args.dry_run)
+        runs = sorted(os.listdir(tool_dir), reverse=True)
+        remove_if_there(runs, 'all-runs')
+        remove_if_there(runs, 'run-list.csv')
+
+        if args.create_toolwise_dirs:
+            # write out run-list.csv
+            with open(os.path.join(tool_dir, 'run-list.csv'), 'w') as fdr:
+                fdr.write('Runs\n')
+                for run in runs:
+                    fdr.write(new_run_name(run) + '\n')
+
+        for run in runs:
+            run_dir = os.path.join(tool_dir, run)
+            global_run_dir = os.path.join(result_dir, run)
+            if not os.path.isdir(run_dir):
+                continue
+            ensure_directory(global_run_dir, args.dry_run)
+            for csvFile in os.listdir(run_dir):
+                # link file in run directory
+                fileToLink = os.path.join(tool_dir, run, csvFile)
+                linkName = os.path.join(global_run_dir, csvFile)
+                create_link(fileToLink, linkName, args.dry_run)
+                # skip if max-size is set and size is exceeded
+                if args.max_size and exceeds(fileToLink, args.max_size):
+                    continue
+                if args.create_toolwise_dirs:
+                    # link in all-runs (rename file accordingly)
+                    match = NAME_PATTERN.match(csvFile)
+                    if match:
+                        if not BENCHMARK_PATTERN.match(csvFile):
+                            tool_name = match.group(2)
+                            copy_replace(fileToLink, all_runs_dir, tool_name, run, args.dry_run)
+                    else:
+                        logging.warn('file did not match pattern: %s', csvFile)
+                # link file in merged directory
+                if csvFile not in already_merged:
+                    linkName = os.path.join(merged_dir_benchmark if BENCHMARK_PATTERN.match(csvFile)
+                                            else merged_dir_individual, csvFile)
+                    create_link(fileToLink, linkName, args.dry_run)
+                    already_merged.append(csvFile)
+                else:
+                    log_action('Skipping %s', csvFile)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Merge results of all benchmark runs.',
+        epilog="""The config file must contain the key "tools" specifying the tools to process.
+        It further can contain any long version of arguments to this program as a default value.
+        Any command line parameter will override such a default value.
+        Additionally, a local version of the file will be read, overriding the default values.
+        Its filename is "local-" prepended to the name of the config file.""")
+    parser.add_argument(
+        "-r", "--result-dir", help="Path to result directory to search in.", type=str)
+    parser.add_argument(
+        "-c", "--create-run-dirs", help="Whether to recreate runs directories.",
+        action="store_true")
+    parser.add_argument(
+        "-t", "--create-toolwise-dirs", help="Whether to recreate toolwise regression directories.",
+        action="store_true")
+    parser.add_argument(
+        "-d", "--dry-run", help="Only print action, don't execute them.",
+        action="store_true")
+    parser.add_argument(
+        "-n", "--no-clean", help="Don't remove previously existing symlinks in merged dir.",
+        dest='clean', action="store_false")
+    parser.add_argument(
+        "-v", "--verbose", help="Print debug messages.", action="store_true")
+    parser.add_argument(
+        "-m", "--max-size", type=int,
+        help="Maximum benchmark size to include. Implies cleaning existing symlinks.")
+    parser.add_argument(
+        "-f", "--file-config", default='merge_results.json', help="Config file to use.")
+    args = parser.parse_args()
+    include_file_config(args)
+    logging.basicConfig(format=FORMAT, level=logging.DEBUG if args.verbose else logging.INFO)
+    main(args)