diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/build.gradle b/trainbenchmark/trainbenchmark-alternate-scripts/build.gradle
index 295ef68a23de21129255c45b90a54e2d1a1ad4a2..fc879099b2d9c4ca1b83fd0cc48db6657cce75ad 100644
--- a/trainbenchmark/trainbenchmark-alternate-scripts/build.gradle
+++ b/trainbenchmark/trainbenchmark-alternate-scripts/build.gradle
@@ -4,36 +4,22 @@ dependencies {
compile project(':trainbenchmark-generator')
compile project(':trainbenchmark-generator-emf')
- compile project(':trainbenchmark-generator-graph-neo4j')
compile project(':trainbenchmark-generator-graph-tinkerpop')
- compile project(':trainbenchmark-generator-rdf')
- compile project(':trainbenchmark-generator-sql')
compile project(':trainbenchmark-generator-json4ag')
- compile project(':trainbenchmark-generator-json4ag-ref')
compile project(':trainbenchmark-generator-dot')
compile project(':trainbenchmark-tool')
-// compile project(':trainbenchmark-tool-blazegraph')
- compile project(':trainbenchmark-tool-drools')
- compile project(':trainbenchmark-tool-eclipseocl')
- compile project(':trainbenchmark-tool-emfapi')
-// compile project(':trainbenchmark-tool-epsilon')
-// compile project(':trainbenchmark-tool-ingraph')
compile project(':trainbenchmark-tool-jastadd-base')
- compile project(':trainbenchmark-tool-jastadd-symbolic-references')
- compile project(':trainbenchmark-tool-jastadd-symbolic-references-base')
- compile project(':trainbenchmark-tool-jastadd-symbolic-references-incremental')
- compile project(':trainbenchmark-tool-jastadd-java-references')
- compile project(':trainbenchmark-tool-jastadd-java-references-base')
- compile project(':trainbenchmark-tool-jastadd-java-references-incremental')
- compile project(':trainbenchmark-tool-jena')
- compile project(':trainbenchmark-tool-kiama')
- compile project(':trainbenchmark-tool-mysql')
- compile project(':trainbenchmark-tool-neo4j')
- compile project(':trainbenchmark-tool-racr-python')
- compile project(':trainbenchmark-tool-racr-scheme')
- compile project(':trainbenchmark-tool-racr-cpp')
- compile project(':trainbenchmark-tool-rdf4j')
- compile project(':trainbenchmark-tool-sqlite')
+ compile project(':trainbenchmark-tool-jastadd-namelookup')
+ compile project(':trainbenchmark-tool-jastadd-namelookup-base')
+ compile project(':trainbenchmark-tool-jastadd-namelookup-incremental')
+ compile project(':trainbenchmark-tool-jastadd-optimized')
+ compile project(':trainbenchmark-tool-jastadd-optimized-base')
+ compile project(':trainbenchmark-tool-jastadd-optimized-incremental')
+// compile project(':trainbenchmark-tool-jastadd-specialized')
+// compile project(':trainbenchmark-tool-jastadd-specialized-base')
+// compile project(':trainbenchmark-tool-jastadd-specialized-incremental')
+// compile project(':trainbenchmark-tool-jastadd-relast')
+// compile project(':trainbenchmark-tool-jastadd-relast-incremental')
compile project(':trainbenchmark-tool-tinkergraph')
compile project(':trainbenchmark-tool-viatra')
}
diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkMain.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkMain.java
index 8659f031381411f57975f3b6dd3582d071a2c46e..c5dca00cfa62bd4fec9cd314ef0672b37949e1a8 100644
--- a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkMain.java
+++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkMain.java
@@ -64,9 +64,13 @@ public class BenchmarkMain {
ObjectMapper mapper = Utils.getMapper();
BenchmarkSettingsBasics bbs = Utils.readFromResource(mapper, "basic-settings.json",
BenchmarkSettingsBasics.class);
+ try {
BenchmarkSettingsBasics loca_bbs = Utils.readFromResource(mapper, "local-basic-settings.json",
BenchmarkSettingsBasics.class);
- Utils.updateBasicsWithLocal(bbs, loca_bbs);
+ Utils.updateBasicsWithLocal(bbs, loca_bbs);
+ } catch (IOException e) {
+ System.out.println("No file 'local-basic-settings.json' found. Using default!");
+ }
// import tools to use
List<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> builders =
diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableFormats.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableFormats.java
index 4b93da1e665b86ec704b109e288dc29c3029b75d..439a0801d10840f94907902fb355864c04a7d921 100644
--- a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableFormats.java
+++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableFormats.java
@@ -1,16 +1,10 @@
package de.tudresden.inf.st.train.scripts;
import de.tudresden.inf.st.trainbenchmark.generator.json4ag.config.Json4AgGeneratorConfigBuilder;
-import de.tudresden.inf.st.trainbenchmark.generator.json4agref.config.Json4AgRefGeneratorConfigBuilder;
import hu.bme.mit.trainbenchmark.generator.config.*;
import hu.bme.mit.trainbenchmark.generator.emf.config.EmfGeneratorConfigBuilder;
-import hu.bme.mit.trainbenchmark.generator.graph.neo4j.config.Neo4jGraphGeneratorConfigBuilder;
import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphFormat;
import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphGeneratorConfigBuilder;
-import hu.bme.mit.trainbenchmark.generator.rdf.config.RdfGeneratorConfigBuilder;
-import hu.bme.mit.trainbenchmark.generator.sql.config.SqlGeneratorConfigBuilder;
-import hu.bme.mit.trainbenchmark.neo4j.config.Neo4jGraphFormat;
-import hu.bme.mit.trainbenchmark.rdf.RdfFormat;
import java.util.*;
import java.util.stream.Collectors;
@@ -45,15 +39,9 @@ public class BenchmarkSettingsAvailableFormats {
private static List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>>
getAllBuilders() {
return Arrays.asList(
- new Neo4jGraphGeneratorConfigBuilder().setGraphFormat(Neo4jGraphFormat.CSV),
- new Neo4jGraphGeneratorConfigBuilder().setGraphFormat(Neo4jGraphFormat.GRAPHML),
new EmfGeneratorConfigBuilder(),
new TinkerGraphGeneratorConfigBuilder().setGraphFormat(TinkerGraphFormat.GRAPHML),
- new Json4AgGeneratorConfigBuilder(),
- new Json4AgRefGeneratorConfigBuilder(),
- new RdfGeneratorConfigBuilder().setFormat(RdfFormat.TURTLE).setInferred(true),
- new RdfGeneratorConfigBuilder().setFormat(RdfFormat.TURTLE).setInferred(false),
- new SqlGeneratorConfigBuilder()
+ new Json4AgGeneratorConfigBuilder()
);
}
diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableTools.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableTools.java
index 1dc230051a123af7a37705099d5e27f21254fc58..49dc323b78a5cbb8f5a12c2e9326b5b3c642f2ab 100644
--- a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableTools.java
+++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableTools.java
@@ -1,23 +1,14 @@
package de.tudresden.inf.st.train.scripts;
-import de.tudresden.inf.st.train.kiama.config.KiamaBenchmarkConfigBuilder;
-import de.tudresden.inf.st.train.racr.config.RacrBenchmarkConfigBuilder;
+import de.tudresden.inf.st.train.jastadd.config.JastaddNameLookupBenchmarkConfigBuilder;
+import de.tudresden.inf.st.train.jastadd.config.JastaddNameLookupIncrementalBenchmarkConfigBuilder;
+import de.tudresden.inf.st.train.jastadd.config.JastaddOptimizedBenchmarkConfigBuilder;
+import de.tudresden.inf.st.train.jastadd.config.JastaddOptimizedIncrementalBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.benchmark.config.*;
import hu.bme.mit.trainbenchmark.benchmark.tinkergraph.config.TinkerGraphBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.benchmark.viatra.config.ViatraBackend;
import hu.bme.mit.trainbenchmark.benchmark.viatra.config.ViatraBenchmarkConfigBuilder;
-import hu.bme.mit.trainbenchmark.benchmark.drools.config.DroolsBenchmarkConfigBuilder;
-import hu.bme.mit.trainbenchmark.benchmark.eclipseocl.config.EclipseOclBenchmarkConfigBuilder;
-import hu.bme.mit.trainbenchmark.benchmark.emfapi.config.EmfApiBenchmarkConfigBuilder;
-import hu.bme.mit.trainbenchmark.benchmark.jena.config.JenaBenchmarkConfigBuilder;
-import hu.bme.mit.trainbenchmark.benchmark.mysql.config.MySqlBenchmarkConfigBuilder;
-import hu.bme.mit.trainbenchmark.benchmark.sqlite.config.SQLiteBenchmarkConfigBuilder;
-import hu.bme.mit.trainbenchmark.benchmark.neo4j.config.Neo4jBenchmarkConfigBuilder;
-import hu.bme.mit.trainbenchmark.neo4j.config.Neo4jGraphFormat;
-import hu.bme.mit.trainbenchmark.benchmark.neo4j.config.Neo4jEngine;
-import hu.bme.mit.trainbenchmark.benchmark.rdf4j.config.Rdf4jBenchmarkConfigBuilder;
-
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -50,24 +41,10 @@ public class BenchmarkSettingsAvailableTools {
private static List<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>>
getAllBuilders() {
return Arrays.asList(
- new DroolsBenchmarkConfigBuilder(),
- new EclipseOclBenchmarkConfigBuilder(),
- new EmfApiBenchmarkConfigBuilder(),
- getBuilderByName("de.tudresden.inf.st.train.jastadd.config.JastaddBenchmarkConfigBuilder"),
- getBuilderByName("de.tudresden.inf.st.train.jastadd.config.JastaddIncrementalBenchmarkConfigBuilder"),
- getBuilderByName("de.tudresden.inf.st.train.jastadd.config.JastaddSymbolicBenchmarkConfigBuilder"),
- getBuilderByName("de.tudresden.inf.st.train.jastadd.config.JastaddSymbolicIncrementalBenchmarkConfigBuilder"),
- new JenaBenchmarkConfigBuilder().setInferencing(false),
- new JenaBenchmarkConfigBuilder().setInferencing(true),
- new KiamaBenchmarkConfigBuilder(),
- new MySqlBenchmarkConfigBuilder(),
- new Neo4jBenchmarkConfigBuilder().setEngine(Neo4jEngine.COREAPI).setGraphFormat(Neo4jGraphFormat.CSV ),
- new Neo4jBenchmarkConfigBuilder().setEngine(Neo4jEngine.CYPHER ).setGraphFormat(Neo4jGraphFormat.GRAPHML),
- new RacrBenchmarkConfigBuilder().setVerbose(false).useCpp().useInternalServer(),
- new RacrBenchmarkConfigBuilder().setVerbose(false).usePython(),
- new RacrBenchmarkConfigBuilder().setVerbose(false).useScheme(),
- new Rdf4jBenchmarkConfigBuilder().setInferencing(false),
- new SQLiteBenchmarkConfigBuilder(),
+ new JastaddNameLookupBenchmarkConfigBuilder(),
+ new JastaddNameLookupIncrementalBenchmarkConfigBuilder(),
+ new JastaddOptimizedBenchmarkConfigBuilder(),
+ new JastaddOptimizedIncrementalBenchmarkConfigBuilder(),
new TinkerGraphBenchmarkConfigBuilder(),
new ViatraBenchmarkConfigBuilder().setBackend(ViatraBackend.INCREMENTAL),
new ViatraBenchmarkConfigBuilder().setBackend(ViatraBackend.LOCAL_SEARCH)
diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/GenerateMain.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/GenerateMain.java
index 052019c495c03eb6b2ee198989f003f32e2151e9..367c5f5f0e3b73014640f242af5df01c4cbd6496 100644
--- a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/GenerateMain.java
+++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/GenerateMain.java
@@ -5,6 +5,7 @@ import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
import hu.bme.mit.trainbenchmark.generator.config.*;
import hu.bme.mit.trainbenchmark.generator.runner.GeneratorRunner;
+import java.io.IOException;
import java.util.List;
/**
@@ -17,9 +18,13 @@ public class GenerateMain {
ObjectMapper mapper = Utils.getMapper();
BenchmarkSettingsBasics bbs = Utils.readFromResource(mapper, "basic-settings.json",
BenchmarkSettingsBasics.class);
- BenchmarkSettingsBasics loca_bbs = Utils.readFromResource(mapper, "local-basic-settings.json",
- BenchmarkSettingsBasics.class);
- Utils.updateBasicsWithLocal(bbs, loca_bbs);
+ try {
+ BenchmarkSettingsBasics loca_bbs = Utils.readFromResource(mapper, "local-basic-settings.json",
+ BenchmarkSettingsBasics.class);
+ Utils.updateBasicsWithLocal(bbs, loca_bbs);
+ } catch (IOException e) {
+ System.out.println("No file 'local-basic-settings.json' found. Using default!");
+ }
// import formats to use
List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>> builders =
diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/basic-settings.json b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/basic-settings.json
index 6d6b000cafc4e090b2ed76e1729522066f9654b1..501ab8fc36d9f4507c97454d9dbc36e37b7637b7 100644
--- a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/basic-settings.json
+++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/basic-settings.json
@@ -5,24 +5,10 @@
"runs": 1,
"dry-run": true,
"tools": [
- "Drools",
- "Eclipse OCL",
- "EMF API",
- "Jena (Inferencing)",
- "Jena (No Inferencing)",
- "MySQL",
- "Neo4j (Core API-CSV)",
- "Neo4j (Cypher-GraphML)",
- "Racr (Scheme)",
- "Racr (Python)",
- "Racr (CPP)",
- "RDF4J (No Inferencing)",
- "SQLite",
- "Jastadd (Symbolic References)",
- "Jastadd (Incremental + Symbolic References)",
- "Jastadd (Java References)",
- "Jastadd (Incremental + Java References)",
- "Kiama",
+ "Jastadd (Name Lookup)",
+ "Jastadd (Incremental Name Lookup)",
+ "Jastadd (Optimized)",
+ "Jastadd (Optimized Incremental)",
"TinkerGraph",
"VIATRA (Incremental)",
"VIATRA (Local Search)"
@@ -30,10 +16,7 @@
"formats": [
"emf",
"json4ag",
- "json4ag-ref",
- "graph-tinkerpop",
- "rdf",
- "sql"
+ "graph-tinkerpop"
],
"jvmSettings": [
{
diff --git a/trainbenchmark/trainbenchmark-config/src/main/java/hu/bme/mit/trainbenchmark/config/ExecutionConfig.java b/trainbenchmark/trainbenchmark-config/src/main/java/hu/bme/mit/trainbenchmark/config/ExecutionConfig.java
index fa0ba9c21b13c1a7e1f5341ecd2a9e23f87f242c..a828d68d792bd8bdea8e49016ac93cea45740d05 100644
--- a/trainbenchmark/trainbenchmark-config/src/main/java/hu/bme/mit/trainbenchmark/config/ExecutionConfig.java
+++ b/trainbenchmark/trainbenchmark-config/src/main/java/hu/bme/mit/trainbenchmark/config/ExecutionConfig.java
@@ -4,14 +4,20 @@ public class ExecutionConfig {
protected Integer initialMemory;
protected Integer maxMemory;
+ protected Boolean dryRun;
- public ExecutionConfig(final Integer initialMemory, final Integer maxMemory) {
+ public ExecutionConfig(final Integer initialMemory, final Integer maxMemory, final boolean dryRun) {
this.initialMemory = initialMemory;
this.maxMemory = maxMemory;
+ this.dryRun = dryRun;
+ }
+
+ public ExecutionConfig(final Integer initialMemory, final Integer maxMemory) {
+ this(initialMemory, maxMemory, true);
}
/**
- *
+ *
* @return The initial memory for the benchmark JVM in MBs.
*/
public Integer getInitialMemory() {
@@ -41,7 +47,15 @@ public class ExecutionConfig {
public String getXmx() {
return maxMemory + "M";
}
-
+
+ /**
+ *
+ * @return whether this benchmark run must not start any Java processes
+ */
+ public Boolean getDryRun() {
+ return dryRun;
+ }
+
public static ExecutionConfig defaultExecutionConfig() {
return new ExecutionConfig(1000, 1000);
}
diff --git a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/config/BenchmarkConfigBase.java b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/config/BenchmarkConfigBase.java
index 8467833190a8872e79dacdf59a40b8923cbeebb8..8ac9f8c43332d12e21fd930026cc3248119238d4 100644
--- a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/config/BenchmarkConfigBase.java
+++ b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/config/BenchmarkConfigBase.java
@@ -24,7 +24,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase {
/**
* The id for the benchmark, used for determining the result directory.
*/
- protected final int benchmarkId;
+ protected final String benchmarkId;
/**
* The timeout for each measurement run in seconds.
@@ -68,7 +68,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase {
*/
protected final Optional<Integer> transformationConstant;
- protected BenchmarkConfigBase(final int benchmarkId, final long timeout, final int runs, final String modelFilename,
+ protected BenchmarkConfigBase(final String benchmarkId, final long timeout, final int runs, final String modelFilename,
final List<RailwayOperation> operations, final String workload,
final TransformationChangeSetStrategy transformationChangeSetStrategy,
final Optional<Integer> queryTransformationCount, final Optional<Integer> transformationConstant) {
@@ -84,7 +84,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase {
this.transformationConstant = transformationConstant;
}
- public int getBenchmarkId() {
+ public String getBenchmarkId() {
return benchmarkId;
}
diff --git a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/config/BenchmarkConfigBaseBuilder.java b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/config/BenchmarkConfigBaseBuilder.java
index 03fb69424a80bf657b84135f5034dba2f00c5194..6e5b1d23253344b852bdf14d7ed017b183182adc 100644
--- a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/config/BenchmarkConfigBaseBuilder.java
+++ b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/config/BenchmarkConfigBaseBuilder.java
@@ -8,7 +8,7 @@ import com.google.common.base.Preconditions;
import hu.bme.mit.trainbenchmark.constants.RailwayOperation;
public final class BenchmarkConfigBaseBuilder {
- private Integer benchmarkId;
+ private String benchmarkId;
private Long timeout;
private Integer runs;
private String modelFilename;
@@ -65,7 +65,7 @@ public final class BenchmarkConfigBaseBuilder {
return this;
}
- public BenchmarkConfigBaseBuilder setBenchmarkId(final Integer benchmarkId) {
+ public BenchmarkConfigBaseBuilder setBenchmarkId(final String benchmarkId) {
this.benchmarkId = benchmarkId;
return this;
}
diff --git a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/result/AbstractResult.java b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/result/AbstractResult.java
index a8143f187ebb36a11ba38191840fdff32191b03d..f2a13294252520192b52c0b2a544adca7c009310 100644
--- a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/result/AbstractResult.java
+++ b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/result/AbstractResult.java
@@ -2,6 +2,7 @@ package hu.bme.mit.trainbenchmark.benchmark.result;
import java.io.File;
import java.io.IOException;
+import java.nio.charset.Charset;
import org.apache.commons.io.FileUtils;
@@ -24,8 +25,9 @@ public abstract class AbstractResult {
protected final String MODEL = "Model";
protected final String RUN = "Run";
- protected final int benchmarkId;
+ protected final String benchmarkId;
protected final String toolName;
+ protected final String projectName;
protected final String workload;
protected final String workspaceDir;
protected final String model;
@@ -34,6 +36,7 @@ public abstract class AbstractResult {
public AbstractResult(final BenchmarkConfig bc) {
this.benchmarkId = bc.getConfigBase().getBenchmarkId();
this.toolName = bc.getToolName();
+ this.projectName = bc.getProjectName();
this.workload = bc.getConfigBase().getWorkload();
this.workspaceDir = bc.getConfigBase().getWorkspaceDir();
this.model = bc.getConfigBase().getModelFilename();
@@ -41,13 +44,15 @@ public abstract class AbstractResult {
}
public void serializeCsv(final String csv, final String filePrefix) throws IOException {
- final String matchesCsvPath = String.format("%s/%s-%s-%s-%s-%s.csv", getResultDir(), filePrefix, toolName,
- workload, model, description);
- FileUtils.write(new File(matchesCsvPath), csv);
+ File targetDir = ResultHelper.getResultDirectory(projectName, benchmarkId);
+ String fileName = String.format("%s-%s-%s-%s-%s.csv", filePrefix, toolName, workload, model, description);
+// final String matchesCsvPath = String.format("%s/%s-%s-%s-%s-%s.csv", getResultDir(), filePrefix, toolName,
+// workload, model, description);
+ FileUtils.write(FileUtils.getFile(targetDir, fileName), csv, Charset.defaultCharset());
}
- public String getResultDir() {
- return workspaceDir + ResultHelper.getResultDirForId(benchmarkId);
- }
+// public String getResultDir() {
+// return workspaceDir + ResultHelper.getResultDirForId(benchmarkId);
+// }
}
diff --git a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/result/ResultHelper.java b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/result/ResultHelper.java
index 56ca0bbedf4cbe7ffd982a478514efb1aec758d2..737335677b1848f11ccc2e48259ffbeca4e9a694 100644
--- a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/result/ResultHelper.java
+++ b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/result/ResultHelper.java
@@ -3,8 +3,15 @@ package hu.bme.mit.trainbenchmark.benchmark.result;
import java.io.File;
import java.io.IOException;
import java.nio.file.*;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
+import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfig;
+import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfigBuilder;
+import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
@@ -12,6 +19,7 @@ public final class ResultHelper {
public static final String WORKSPACE_DIR = "../";
public static final String RESULT_DIR = "results/";
+ public static final String TOOLS_DIR = "tools/";
public static final String DIAGRAM_DIR = "diagrams/";
public static final String SCRIPT_PATH = WORKSPACE_DIR + "trainbenchmark-scripts/src/";
public static final String BENCHMARK_SCRIPT = "BenchmarkScript.groovy";
@@ -94,4 +102,38 @@ public final class ResultHelper {
FileUtils.copyFile(srcFile, destFile);
}
+ public static void createNewResultDirs(Collection<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> tools) {
+ }
+
+ /**
+ * Prepare a benchmark run, i.e., creates necessary directories. This operation is idempotent.
+ * @param config the benchmark configuration
+ * @param ec the execution configuration
+ */
+ public static void prepare(BenchmarkConfig config, ExecutionConfig ec) {
+ // ensure directory for the tool
+ File toolDir = getToolDirectory(config.getProjectName());
+ toolDir.mkdir();
+ // ensure directory for the run inside toolDir
+ File runDir = getResultDirectory(toolDir, config.getConfigBase().getBenchmarkId());
+ runDir.mkdir();
+ }
+
+ public static String getNow() {
+ DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
+ return dateFormat.format(new Date());
+ }
+
+ public static File getToolDirectory(String projectName) {
+ return FileUtils.getFile(WORKSPACE_DIR, RESULT_DIR, TOOLS_DIR, projectName);
+ }
+
+ public static File getResultDirectory(File toolDirectory, String benchmarkId) {
+ return FileUtils.getFile(toolDirectory, "run-" + benchmarkId);
+ }
+
+ public static File getResultDirectory(String projectName, String benchmarkId) {
+ return getResultDirectory(getToolDirectory(projectName), benchmarkId);
+ }
+
}
diff --git a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/runcomponents/BenchmarkRunner.java b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/runcomponents/BenchmarkRunner.java
index 9136edcca823a519a228c863d31ec4b1aaada6c4..06d8bb80862e12e22e66e9c30b3742560367c0b4 100644
--- a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/runcomponents/BenchmarkRunner.java
+++ b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/runcomponents/BenchmarkRunner.java
@@ -11,14 +11,38 @@ import org.apache.commons.exec.Executor;
import org.apache.commons.exec.PumpStreamHandler;
import com.google.common.base.Joiner;
-
+import com.google.common.collect.ImmutableMap;
import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfig;
import hu.bme.mit.trainbenchmark.benchmark.memory.MemoryResult;
import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
+import org.apache.commons.exec.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Map;
public class BenchmarkRunner {
- public static int runPerformanceBenchmark(final BenchmarkConfig bc, final ExecutionConfig ec)
+ public static String dictFormat(String format, Map<String, Object> values) {
+ StringBuilder convFormat = new StringBuilder(format);
+ ArrayList<Object> valueList = new ArrayList<>();
+ int currentPos = 1;
+ for (String key : values.keySet()) {
+ String formatKey = "%(" + key + ")",
+ formatPos = "%" + Integer.toString(currentPos) + "$1s";
+ int index = -1;
+ while ((index = convFormat.indexOf(formatKey, index)) != -1) {
+ convFormat.replace(index, index + formatKey.length(), formatPos);
+ index += formatPos.length();
+ }
+ valueList.add(values.get(key));
+ ++currentPos;
+ }
+ return String.format(convFormat.toString(), valueList.toArray());
+ }
+
+ public static int runPerformanceBenchmark(final BenchmarkConfig bc, final ExecutionConfig ec, String jvmSetting)
throws IOException, InterruptedException {
final Joiner joiner = Joiner.on(", ");
System.out.println("Running benchmark.");
@@ -38,9 +62,17 @@ public class BenchmarkRunner {
final String jarPath = String.format("../%s/build/libs/%s-1.0.0-SNAPSHOT-fat.jar %s", projectName, projectName,
configPath);
- final String javaCommand = String.format("java -Xms%s -Xmx%s -server -Xverify:none -jar %s %s", ec.getXms(), ec.getXmx(), jarPath, configPath);
+ final String javaCommand = dictFormat("java " + jvmSetting + " -jar %(jarPath) %(configPath)",
+ ImmutableMap.of("Xms", ec.getXms(), "Xmx", ec.getXmx(),
+ "jarPath", jarPath, "configPath", configPath));
+
+// final String javaCommand = String.format("java -Xms%s -Xmx%s -server -Xverify:none -jar %s %s", ec.getXms(), ec.getXmx(), jarPath, configPath);
// final String javaCommand = String.format("java -Xms%s -Xmx%s -server -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:StartFlightRecording=dumponexit=true -Xverify:none -jar %s %s", ec.getXms(), ec.getXmx(), jarPath, configPath);
final CommandLine cmdLine = CommandLine.parse(javaCommand);
+ if (ec.getDryRun()) {
+ System.out.println("Would have executed: "+ cmdLine.toString());
+ return 0;
+ }
final long timeoutInSeconds = bc.getConfigBase().getTimeout();
final long timeoutInMilliseconds = timeoutInSeconds * 1000;
@@ -63,7 +95,7 @@ public class BenchmarkRunner {
}
public static int runMemoryBenchmark(final BenchmarkConfig bc, final ExecutionConfig defaultEc,
- final int numberOfSteps) throws IOException, InterruptedException {
+ final int numberOfSteps, String jvmSetting) throws IOException, InterruptedException {
// e.g. initialMaxMemory = 12800, we save this (as a final variable), so
// that we will not exceed it
//
@@ -79,7 +111,7 @@ public class BenchmarkRunner {
memoryQuantum /= 2;
final ExecutionConfig ec = new ExecutionConfig(currentMaxMemory, currentMaxMemory);
- if (runPerformanceBenchmark(bc, ec) == 0) {
+ if (runPerformanceBenchmark(bc, ec, jvmSetting) == 0) {
System.out.println("Execution finished, testing with less memory.");
System.out.println();
currentMaxMemory -= memoryQuantum;
diff --git a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/test/TrainBenchmarkTest.java b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/test/TrainBenchmarkTest.java
index da4721781f6fffb5136e7780d2cd41f6da93c6da..d8ac00dd5b0e3464cae1cc1d919e46ad59bca0eb 100644
--- a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/test/TrainBenchmarkTest.java
+++ b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/test/TrainBenchmarkTest.java
@@ -23,7 +23,7 @@ public abstract class TrainBenchmarkTest {
@Rule
public ErrorCollector collector = new ErrorCollector();
- protected final int benchmarkId = 0;
+ protected final String benchmarkId = "";
protected ExecutionConfig executionConfig = ExecutionConfig.defaultExecutionConfig();
protected final long timeout = 120;
protected final int runs = 1;
diff --git a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/test/queryspecific/QueryTest.java b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/test/queryspecific/QueryTest.java
index 4046712c480f14d106115bbea029fe14539d997b..f616b63636e4b2c5cd8a827177fb43b092a29da7 100644
--- a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/test/queryspecific/QueryTest.java
+++ b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/test/queryspecific/QueryTest.java
@@ -11,7 +11,7 @@ import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
public abstract class QueryTest {
- protected final int benchmarkId = 0;
+ protected final String benchmarkId = "";
protected ExecutionConfig executionConfig = ExecutionConfig.defaultExecutionConfig();
protected final long timeout = 120;
protected final int runs = 1;