Make alternate scripts work again.

- Excluded JastAdd Specialized and JastAdd Relast because of build problems

trainbenchmark/trainbenchmark-alternate-scripts/build.gradle

@@ -4,36 +4,22 @@ dependencies {
 
 	compile project(':trainbenchmark-generator')
 	compile project(':trainbenchmark-generator-emf')
-	compile project(':trainbenchmark-generator-graph-neo4j')
 	compile project(':trainbenchmark-generator-graph-tinkerpop')
-	compile project(':trainbenchmark-generator-rdf')
-	compile project(':trainbenchmark-generator-sql')
 	compile project(':trainbenchmark-generator-json4ag')
-	compile project(':trainbenchmark-generator-json4ag-ref')
 	compile project(':trainbenchmark-generator-dot')
 	compile project(':trainbenchmark-tool')
-//	compile project(':trainbenchmark-tool-blazegraph')
-	compile project(':trainbenchmark-tool-drools')
-	compile project(':trainbenchmark-tool-eclipseocl')
-	compile project(':trainbenchmark-tool-emfapi')
-//	compile project(':trainbenchmark-tool-epsilon')
-//	compile project(':trainbenchmark-tool-ingraph')
 	compile project(':trainbenchmark-tool-jastadd-base')
-	compile project(':trainbenchmark-tool-jastadd-symbolic-references')
-	compile project(':trainbenchmark-tool-jastadd-symbolic-references-base')
-	compile project(':trainbenchmark-tool-jastadd-symbolic-references-incremental')
-	compile project(':trainbenchmark-tool-jastadd-java-references')
-	compile project(':trainbenchmark-tool-jastadd-java-references-base')
-	compile project(':trainbenchmark-tool-jastadd-java-references-incremental')
-	compile project(':trainbenchmark-tool-jena')
-	compile project(':trainbenchmark-tool-kiama')
-	compile project(':trainbenchmark-tool-mysql')
-	compile project(':trainbenchmark-tool-neo4j')
-	compile project(':trainbenchmark-tool-racr-python')
-	compile project(':trainbenchmark-tool-racr-scheme')
-	compile project(':trainbenchmark-tool-racr-cpp')
-	compile project(':trainbenchmark-tool-rdf4j')
-	compile project(':trainbenchmark-tool-sqlite')
+	compile project(':trainbenchmark-tool-jastadd-namelookup')
+	compile project(':trainbenchmark-tool-jastadd-namelookup-base')
+	compile project(':trainbenchmark-tool-jastadd-namelookup-incremental')
+	compile project(':trainbenchmark-tool-jastadd-optimized')
+	compile project(':trainbenchmark-tool-jastadd-optimized-base')
+	compile project(':trainbenchmark-tool-jastadd-optimized-incremental')
+//	compile project(':trainbenchmark-tool-jastadd-specialized')
+//	compile project(':trainbenchmark-tool-jastadd-specialized-base')
+//	compile project(':trainbenchmark-tool-jastadd-specialized-incremental')
+//	compile project(':trainbenchmark-tool-jastadd-relast')
+//	compile project(':trainbenchmark-tool-jastadd-relast-incremental')
 	compile project(':trainbenchmark-tool-tinkergraph')
 	compile project(':trainbenchmark-tool-viatra')
 }

trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkMain.java

@@ -64,9 +64,13 @@ public class BenchmarkMain {
 		ObjectMapper mapper = Utils.getMapper();
 		BenchmarkSettingsBasics bbs = Utils.readFromResource(mapper, "basic-settings.json",
 				BenchmarkSettingsBasics.class);
+		try {
 		BenchmarkSettingsBasics loca_bbs = Utils.readFromResource(mapper, "local-basic-settings.json",
 				BenchmarkSettingsBasics.class);
-		Utils.updateBasicsWithLocal(bbs, loca_bbs);
+			Utils.updateBasicsWithLocal(bbs, loca_bbs);
+		} catch (IOException e) {
+			System.out.println("No file 'local-basic-settings.json' found. Using default!");
+		}
 
 		// import tools to use
 		List<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> builders =

trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableFormats.java

 package de.tudresden.inf.st.train.scripts;
 
 import de.tudresden.inf.st.trainbenchmark.generator.json4ag.config.Json4AgGeneratorConfigBuilder;
-import de.tudresden.inf.st.trainbenchmark.generator.json4agref.config.Json4AgRefGeneratorConfigBuilder;
 import hu.bme.mit.trainbenchmark.generator.config.*;
 import hu.bme.mit.trainbenchmark.generator.emf.config.EmfGeneratorConfigBuilder;
-import hu.bme.mit.trainbenchmark.generator.graph.neo4j.config.Neo4jGraphGeneratorConfigBuilder;
 import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphFormat;
 import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphGeneratorConfigBuilder;
-import hu.bme.mit.trainbenchmark.generator.rdf.config.RdfGeneratorConfigBuilder;
-import hu.bme.mit.trainbenchmark.generator.sql.config.SqlGeneratorConfigBuilder;
-import hu.bme.mit.trainbenchmark.neo4j.config.Neo4jGraphFormat;
-import hu.bme.mit.trainbenchmark.rdf.RdfFormat;
 
 import java.util.*;
 import java.util.stream.Collectors;
@@ -45,15 +39,9 @@ public class BenchmarkSettingsAvailableFormats {
 	private static List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>>
 	getAllBuilders() {
 		return Arrays.asList(
-				new Neo4jGraphGeneratorConfigBuilder().setGraphFormat(Neo4jGraphFormat.CSV),
-				new Neo4jGraphGeneratorConfigBuilder().setGraphFormat(Neo4jGraphFormat.GRAPHML),
 				new EmfGeneratorConfigBuilder(),
 				new TinkerGraphGeneratorConfigBuilder().setGraphFormat(TinkerGraphFormat.GRAPHML),
-				new Json4AgGeneratorConfigBuilder(),
-				new Json4AgRefGeneratorConfigBuilder(),
-				new RdfGeneratorConfigBuilder().setFormat(RdfFormat.TURTLE).setInferred(true),
-				new RdfGeneratorConfigBuilder().setFormat(RdfFormat.TURTLE).setInferred(false),
-				new SqlGeneratorConfigBuilder()
+				new Json4AgGeneratorConfigBuilder()
 		);
 	}
 

trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableTools.java

 package de.tudresden.inf.st.train.scripts;
 
-import de.tudresden.inf.st.train.kiama.config.KiamaBenchmarkConfigBuilder;
-import de.tudresden.inf.st.train.racr.config.RacrBenchmarkConfigBuilder;
+import de.tudresden.inf.st.train.jastadd.config.JastaddNameLookupBenchmarkConfigBuilder;
+import de.tudresden.inf.st.train.jastadd.config.JastaddNameLookupIncrementalBenchmarkConfigBuilder;
+import de.tudresden.inf.st.train.jastadd.config.JastaddOptimizedBenchmarkConfigBuilder;
+import de.tudresden.inf.st.train.jastadd.config.JastaddOptimizedIncrementalBenchmarkConfigBuilder;
 import hu.bme.mit.trainbenchmark.benchmark.config.*;
 import hu.bme.mit.trainbenchmark.benchmark.tinkergraph.config.TinkerGraphBenchmarkConfigBuilder;
 import hu.bme.mit.trainbenchmark.benchmark.viatra.config.ViatraBackend;
 import hu.bme.mit.trainbenchmark.benchmark.viatra.config.ViatraBenchmarkConfigBuilder;
 
-import hu.bme.mit.trainbenchmark.benchmark.drools.config.DroolsBenchmarkConfigBuilder;
-import hu.bme.mit.trainbenchmark.benchmark.eclipseocl.config.EclipseOclBenchmarkConfigBuilder;
-import hu.bme.mit.trainbenchmark.benchmark.emfapi.config.EmfApiBenchmarkConfigBuilder;
-import hu.bme.mit.trainbenchmark.benchmark.jena.config.JenaBenchmarkConfigBuilder;
-import hu.bme.mit.trainbenchmark.benchmark.mysql.config.MySqlBenchmarkConfigBuilder;
-import hu.bme.mit.trainbenchmark.benchmark.sqlite.config.SQLiteBenchmarkConfigBuilder;
-import hu.bme.mit.trainbenchmark.benchmark.neo4j.config.Neo4jBenchmarkConfigBuilder;
-import hu.bme.mit.trainbenchmark.neo4j.config.Neo4jGraphFormat;
-import hu.bme.mit.trainbenchmark.benchmark.neo4j.config.Neo4jEngine;
-import hu.bme.mit.trainbenchmark.benchmark.rdf4j.config.Rdf4jBenchmarkConfigBuilder;
-
 import java.util.*;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -50,24 +41,10 @@ public class BenchmarkSettingsAvailableTools {
 	private static List<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>>
 	getAllBuilders() {
 		return Arrays.asList(
-			    new DroolsBenchmarkConfigBuilder(),
-			    new EclipseOclBenchmarkConfigBuilder(),
-				new EmfApiBenchmarkConfigBuilder(),
-				getBuilderByName("de.tudresden.inf.st.train.jastadd.config.JastaddBenchmarkConfigBuilder"),
-				getBuilderByName("de.tudresden.inf.st.train.jastadd.config.JastaddIncrementalBenchmarkConfigBuilder"),
-				getBuilderByName("de.tudresden.inf.st.train.jastadd.config.JastaddSymbolicBenchmarkConfigBuilder"),
-				getBuilderByName("de.tudresden.inf.st.train.jastadd.config.JastaddSymbolicIncrementalBenchmarkConfigBuilder"),
-				new JenaBenchmarkConfigBuilder().setInferencing(false),
-				new JenaBenchmarkConfigBuilder().setInferencing(true),
-				new KiamaBenchmarkConfigBuilder(),
-				new MySqlBenchmarkConfigBuilder(),
-				new Neo4jBenchmarkConfigBuilder().setEngine(Neo4jEngine.COREAPI).setGraphFormat(Neo4jGraphFormat.CSV    ),
-				new Neo4jBenchmarkConfigBuilder().setEngine(Neo4jEngine.CYPHER ).setGraphFormat(Neo4jGraphFormat.GRAPHML),
-				new RacrBenchmarkConfigBuilder().setVerbose(false).useCpp().useInternalServer(),
-				new RacrBenchmarkConfigBuilder().setVerbose(false).usePython(),
-				new	RacrBenchmarkConfigBuilder().setVerbose(false).useScheme(),
-				new Rdf4jBenchmarkConfigBuilder().setInferencing(false),
-				new SQLiteBenchmarkConfigBuilder(),
+				new JastaddNameLookupBenchmarkConfigBuilder(),
+				new JastaddNameLookupIncrementalBenchmarkConfigBuilder(),
+				new JastaddOptimizedBenchmarkConfigBuilder(),
+				new JastaddOptimizedIncrementalBenchmarkConfigBuilder(),
 				new TinkerGraphBenchmarkConfigBuilder(),
 				new ViatraBenchmarkConfigBuilder().setBackend(ViatraBackend.INCREMENTAL),
 				new ViatraBenchmarkConfigBuilder().setBackend(ViatraBackend.LOCAL_SEARCH)

trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/GenerateMain.java

@@ -5,6 +5,7 @@ import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
 import hu.bme.mit.trainbenchmark.generator.config.*;
 import hu.bme.mit.trainbenchmark.generator.runner.GeneratorRunner;
 
+import java.io.IOException;
 import java.util.List;
 
 /**
@@ -17,9 +18,13 @@ public class GenerateMain {
 		ObjectMapper mapper = Utils.getMapper();
 		BenchmarkSettingsBasics bbs = Utils.readFromResource(mapper, "basic-settings.json",
 				BenchmarkSettingsBasics.class);
-		BenchmarkSettingsBasics loca_bbs = Utils.readFromResource(mapper, "local-basic-settings.json",
-				BenchmarkSettingsBasics.class);
-		Utils.updateBasicsWithLocal(bbs, loca_bbs);
+		try {
+			BenchmarkSettingsBasics loca_bbs = Utils.readFromResource(mapper, "local-basic-settings.json",
+					BenchmarkSettingsBasics.class);
+			Utils.updateBasicsWithLocal(bbs, loca_bbs);
+		} catch (IOException e) {
+			System.out.println("No file 'local-basic-settings.json' found. Using default!");
+		}
 
 		// import formats to use
 		List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>> builders =

trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/basic-settings.json

@@ -5,24 +5,10 @@
   "runs": 1,
   "dry-run": true,
   "tools": [
-    "Drools",
-    "Eclipse OCL",
-    "EMF API",
-    "Jena (Inferencing)",
-    "Jena (No Inferencing)",
-    "MySQL",
-    "Neo4j (Core API-CSV)",
-    "Neo4j (Cypher-GraphML)",
-    "Racr (Scheme)",
-    "Racr (Python)",
-    "Racr (CPP)",
-    "RDF4J (No Inferencing)",
-    "SQLite",
-    "Jastadd (Symbolic References)",
-    "Jastadd (Incremental + Symbolic References)",
-    "Jastadd (Java References)",
-    "Jastadd (Incremental + Java References)",
-    "Kiama",
+    "Jastadd (Name Lookup)",
+    "Jastadd (Incremental Name Lookup)",
+    "Jastadd (Optimized)",
+    "Jastadd (Optimized Incremental)",
     "TinkerGraph",
     "VIATRA (Incremental)",
     "VIATRA (Local Search)"
@@ -30,10 +16,7 @@
   "formats": [
     "emf",
     "json4ag",
-    "json4ag-ref",
-    "graph-tinkerpop",
-    "rdf",
-    "sql"
+    "graph-tinkerpop"
   ],
   "jvmSettings": [
     {

trainbenchmark/trainbenchmark-config/src/main/java/hu/bme/mit/trainbenchmark/config/ExecutionConfig.java

@@ -4,14 +4,20 @@ public class ExecutionConfig {
 
 	protected Integer initialMemory;
 	protected Integer maxMemory;
+	protected Boolean dryRun;
 	
-	public ExecutionConfig(final Integer initialMemory, final Integer maxMemory) {
+	public ExecutionConfig(final Integer initialMemory, final Integer maxMemory, final boolean dryRun) {
 		this.initialMemory = initialMemory;
 		this.maxMemory = maxMemory;
+		this.dryRun = dryRun;
+	}
+
+	public ExecutionConfig(final Integer initialMemory, final Integer maxMemory) {
+		this(initialMemory, maxMemory, true);
 	}
 
 	/**
-	 * 
+	 *
 	 * @return The initial memory for the benchmark JVM in MBs.
 	 */
 	public Integer getInitialMemory() {
@@ -41,7 +47,15 @@ public class ExecutionConfig {
 	public String getXmx() {
 		return maxMemory + "M";
 	}
-	
+
+	/**
+	 *
+	 * @return whether this benchmark run must not start any Java processes
+	 */
+	public Boolean getDryRun() {
+		return dryRun;
+	}
+
 	public static ExecutionConfig defaultExecutionConfig() {
 		return new ExecutionConfig(1000, 1000);
 	}

trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/config/BenchmarkConfigBase.java

@@ -24,7 +24,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase {
 	/**
 	 * The id for the benchmark, used for determining the result directory.
 	 */
-	protected final int benchmarkId;
+	protected final String benchmarkId;
 
 	/**
 	 * The timeout for each measurement run in seconds.
@@ -68,7 +68,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase {
 	 */
 	protected final Optional<Integer> transformationConstant;
 
-	protected BenchmarkConfigBase(final int benchmarkId, final long timeout, final int runs, final String modelFilename,
+	protected BenchmarkConfigBase(final String benchmarkId, final long timeout, final int runs, final String modelFilename,
 			final List<RailwayOperation> operations, final String workload,
 			final TransformationChangeSetStrategy transformationChangeSetStrategy,
 			final Optional<Integer> queryTransformationCount, final Optional<Integer> transformationConstant) {
@@ -84,7 +84,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase {
 		this.transformationConstant = transformationConstant;
 	}
 
-	public int getBenchmarkId() {
+	public String getBenchmarkId() {
 		return benchmarkId;
 	}
 

trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/config/BenchmarkConfigBaseBuilder.java

@@ -8,7 +8,7 @@ import com.google.common.base.Preconditions;
 import hu.bme.mit.trainbenchmark.constants.RailwayOperation;
 
 public final class BenchmarkConfigBaseBuilder {
-	private Integer benchmarkId;
+	private String benchmarkId;
 	private Long timeout;
 	private Integer runs;
 	private String modelFilename;
@@ -65,7 +65,7 @@ public final class BenchmarkConfigBaseBuilder {
 		return this;
 	}
 
-	public BenchmarkConfigBaseBuilder setBenchmarkId(final Integer benchmarkId) {
+	public BenchmarkConfigBaseBuilder setBenchmarkId(final String benchmarkId) {
 		this.benchmarkId = benchmarkId;
 		return this;
 	}

trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/result/AbstractResult.java

@@ -2,6 +2,7 @@ package hu.bme.mit.trainbenchmark.benchmark.result;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.charset.Charset;
 
 import org.apache.commons.io.FileUtils;
 
@@ -24,8 +25,9 @@ public abstract class AbstractResult {
 	protected final String MODEL = "Model";
 	protected final String RUN = "Run";
 
-	protected final int benchmarkId;
+	protected final String benchmarkId;
 	protected final String toolName;
+	protected final String projectName;
 	protected final String workload;
 	protected final String workspaceDir;
 	protected final String model;
@@ -34,6 +36,7 @@ public abstract class AbstractResult {
 	public AbstractResult(final BenchmarkConfig bc) {
 		this.benchmarkId = bc.getConfigBase().getBenchmarkId();
 		this.toolName = bc.getToolName();
+		this.projectName = bc.getProjectName();
 		this.workload = bc.getConfigBase().getWorkload();
 		this.workspaceDir = bc.getConfigBase().getWorkspaceDir();
 		this.model = bc.getConfigBase().getModelFilename();
@@ -41,13 +44,15 @@ public abstract class AbstractResult {
 	}
 
 	public void serializeCsv(final String csv, final String filePrefix) throws IOException {
-		final String matchesCsvPath = String.format("%s/%s-%s-%s-%s-%s.csv", getResultDir(), filePrefix, toolName,
-				workload, model, description);
-		FileUtils.write(new File(matchesCsvPath), csv);
+		File targetDir = ResultHelper.getResultDirectory(projectName, benchmarkId);
+		String fileName = String.format("%s-%s-%s-%s-%s.csv", filePrefix, toolName, workload, model, description);
+//		final String matchesCsvPath = String.format("%s/%s-%s-%s-%s-%s.csv", getResultDir(), filePrefix, toolName,
+//				workload, model, description);
+		FileUtils.write(FileUtils.getFile(targetDir, fileName), csv, Charset.defaultCharset());
 	}
 
-	public String getResultDir() {
-		return workspaceDir + ResultHelper.getResultDirForId(benchmarkId);
-	}
+//	public String getResultDir() {
+//		return workspaceDir + ResultHelper.getResultDirForId(benchmarkId);
+//	}
 
 }

trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/result/ResultHelper.java

@@ -3,8 +3,15 @@ package hu.bme.mit.trainbenchmark.benchmark.result;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.*;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
 import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
 
+import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfig;
+import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfigBuilder;
+import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.StringUtils;
 
@@ -12,6 +19,7 @@ public final class ResultHelper {
 
 	public static final String WORKSPACE_DIR = "../";
 	public static final String RESULT_DIR = "results/";
+	public static final String TOOLS_DIR = "tools/";
 	public static final String DIAGRAM_DIR = "diagrams/";
 	public static final String SCRIPT_PATH = WORKSPACE_DIR + "trainbenchmark-scripts/src/";
 	public static final String BENCHMARK_SCRIPT = "BenchmarkScript.groovy";
@@ -94,4 +102,38 @@ public final class ResultHelper {
 		FileUtils.copyFile(srcFile, destFile);
 	}
 
+	public static void createNewResultDirs(Collection<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> tools) {
+	}
+
+	/**
+	 * Prepare a benchmark run, i.e., creates necessary directories. This operation is idempotent.
+	 * @param config the benchmark configuration
+	 * @param ec the execution configuration
+	 */
+	public static void prepare(BenchmarkConfig config, ExecutionConfig ec) {
+		// ensure directory for the tool
+		File toolDir = getToolDirectory(config.getProjectName());
+		toolDir.mkdir();
+		// ensure directory for the run inside toolDir
+		File runDir = getResultDirectory(toolDir, config.getConfigBase().getBenchmarkId());
+		runDir.mkdir();
+	}
+
+	public static String getNow() {
+		DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
+		return dateFormat.format(new Date());
+	}
+
+	public static File getToolDirectory(String projectName) {
+		return FileUtils.getFile(WORKSPACE_DIR, RESULT_DIR, TOOLS_DIR, projectName);
+	}
+
+	public static File getResultDirectory(File toolDirectory, String benchmarkId) {
+		return FileUtils.getFile(toolDirectory, "run-" + benchmarkId);
+	}
+
+	public static File getResultDirectory(String projectName, String benchmarkId) {
+		return getResultDirectory(getToolDirectory(projectName), benchmarkId);
+	}
+
 }

trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/runcomponents/BenchmarkRunner.java

@@ -11,14 +11,38 @@ import org.apache.commons.exec.Executor;
 import org.apache.commons.exec.PumpStreamHandler;
 
 import com.google.common.base.Joiner;
-
+import com.google.common.collect.ImmutableMap;
 import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfig;
 import hu.bme.mit.trainbenchmark.benchmark.memory.MemoryResult;
 import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
+import org.apache.commons.exec.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Map;
 
 public class BenchmarkRunner {
 
-	public static int runPerformanceBenchmark(final BenchmarkConfig bc, final ExecutionConfig ec)
+	public static String dictFormat(String format, Map<String, Object> values) {
+		StringBuilder convFormat = new StringBuilder(format);
+		ArrayList<Object> valueList = new ArrayList<>();
+		int currentPos = 1;
+		for (String key : values.keySet()) {
+			String formatKey = "%(" + key + ")",
+					formatPos = "%" + Integer.toString(currentPos) + "$1s";
+			int index = -1;
+			while ((index = convFormat.indexOf(formatKey, index)) != -1) {
+				convFormat.replace(index, index + formatKey.length(), formatPos);
+				index += formatPos.length();
+			}
+			valueList.add(values.get(key));
+			++currentPos;
+		}
+		return String.format(convFormat.toString(), valueList.toArray());
+	}
+
+	public static int runPerformanceBenchmark(final BenchmarkConfig bc, final ExecutionConfig ec, String jvmSetting)
 			throws IOException, InterruptedException {
 		final Joiner joiner = Joiner.on(", ");
 		System.out.println("Running benchmark.");
@@ -38,9 +62,17 @@ public class BenchmarkRunner {
 		final String jarPath = String.format("../%s/build/libs/%s-1.0.0-SNAPSHOT-fat.jar %s", projectName, projectName,
 				configPath);
 
-		final String javaCommand = String.format("java -Xms%s -Xmx%s -server -Xverify:none -jar %s %s", ec.getXms(), ec.getXmx(), jarPath, configPath);
+		final String javaCommand = dictFormat("java " + jvmSetting + " -jar %(jarPath) %(configPath)",
+				ImmutableMap.of("Xms", ec.getXms(), "Xmx", ec.getXmx(),
+						"jarPath", jarPath, "configPath", configPath));
+
+//		final String javaCommand = String.format("java -Xms%s -Xmx%s -server -Xverify:none -jar %s %s", ec.getXms(), ec.getXmx(), jarPath, configPath);
 //		final String javaCommand = String.format("java -Xms%s -Xmx%s -server -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:StartFlightRecording=dumponexit=true -Xverify:none -jar %s %s", ec.getXms(), ec.getXmx(), jarPath, configPath);
 		final CommandLine cmdLine = CommandLine.parse(javaCommand);
+		if (ec.getDryRun()) {
+			System.out.println("Would have executed: "+ cmdLine.toString());
+			return 0;
+		}
 
 		final long timeoutInSeconds = bc.getConfigBase().getTimeout();
 		final long timeoutInMilliseconds = timeoutInSeconds * 1000;
@@ -63,7 +95,7 @@ public class BenchmarkRunner {
 	}
 
 	public static int runMemoryBenchmark(final BenchmarkConfig bc, final ExecutionConfig defaultEc,
-			final int numberOfSteps) throws IOException, InterruptedException {
+			final int numberOfSteps, String jvmSetting) throws IOException, InterruptedException {
 		// e.g. initialMaxMemory = 12800, we save this (as a final variable), so
 		// that we will not exceed it
 		//
@@ -79,7 +111,7 @@ public class BenchmarkRunner {
 			memoryQuantum /= 2;
 
 			final ExecutionConfig ec = new ExecutionConfig(currentMaxMemory, currentMaxMemory);
-			if (runPerformanceBenchmark(bc, ec) == 0) {
+			if (runPerformanceBenchmark(bc, ec, jvmSetting) == 0) {
 				System.out.println("Execution finished, testing with less memory.");
 				System.out.println();
 				currentMaxMemory -= memoryQuantum;

trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/test/TrainBenchmarkTest.java

@@ -23,7 +23,7 @@ public abstract class TrainBenchmarkTest {
 	@Rule
 	public ErrorCollector collector = new ErrorCollector();
 
-	protected final int benchmarkId = 0;
+	protected final String benchmarkId = "";
 	protected ExecutionConfig executionConfig = ExecutionConfig.defaultExecutionConfig();
 	protected final long timeout = 120;
 	protected final int runs = 1;

trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/test/queryspecific/QueryTest.java

@@ -11,7 +11,7 @@ import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
 
 public abstract class QueryTest {
 
-	protected final int benchmarkId = 0;
+	protected final String benchmarkId = "";
 	protected ExecutionConfig executionConfig = ExecutionConfig.defaultExecutionConfig();
 	protected final long timeout = 120;
 	protected final int runs = 1;