Skip to content
Snippets Groups Projects
Commit 70286d09 authored by René Schöne's avatar René Schöne
Browse files

Make alternate scripts work again.

- Excluded JastAdd Specialized and JastAdd Relast because of build problems
parent 85efe24f
Branches
No related tags found
No related merge requests found
Showing
with 153 additions and 117 deletions
...@@ -4,36 +4,22 @@ dependencies { ...@@ -4,36 +4,22 @@ dependencies {
compile project(':trainbenchmark-generator') compile project(':trainbenchmark-generator')
compile project(':trainbenchmark-generator-emf') compile project(':trainbenchmark-generator-emf')
compile project(':trainbenchmark-generator-graph-neo4j')
compile project(':trainbenchmark-generator-graph-tinkerpop') compile project(':trainbenchmark-generator-graph-tinkerpop')
compile project(':trainbenchmark-generator-rdf')
compile project(':trainbenchmark-generator-sql')
compile project(':trainbenchmark-generator-json4ag') compile project(':trainbenchmark-generator-json4ag')
compile project(':trainbenchmark-generator-json4ag-ref')
compile project(':trainbenchmark-generator-dot') compile project(':trainbenchmark-generator-dot')
compile project(':trainbenchmark-tool') compile project(':trainbenchmark-tool')
// compile project(':trainbenchmark-tool-blazegraph')
compile project(':trainbenchmark-tool-drools')
compile project(':trainbenchmark-tool-eclipseocl')
compile project(':trainbenchmark-tool-emfapi')
// compile project(':trainbenchmark-tool-epsilon')
// compile project(':trainbenchmark-tool-ingraph')
compile project(':trainbenchmark-tool-jastadd-base') compile project(':trainbenchmark-tool-jastadd-base')
compile project(':trainbenchmark-tool-jastadd-symbolic-references') compile project(':trainbenchmark-tool-jastadd-namelookup')
compile project(':trainbenchmark-tool-jastadd-symbolic-references-base') compile project(':trainbenchmark-tool-jastadd-namelookup-base')
compile project(':trainbenchmark-tool-jastadd-symbolic-references-incremental') compile project(':trainbenchmark-tool-jastadd-namelookup-incremental')
compile project(':trainbenchmark-tool-jastadd-java-references') compile project(':trainbenchmark-tool-jastadd-optimized')
compile project(':trainbenchmark-tool-jastadd-java-references-base') compile project(':trainbenchmark-tool-jastadd-optimized-base')
compile project(':trainbenchmark-tool-jastadd-java-references-incremental') compile project(':trainbenchmark-tool-jastadd-optimized-incremental')
compile project(':trainbenchmark-tool-jena') // compile project(':trainbenchmark-tool-jastadd-specialized')
compile project(':trainbenchmark-tool-kiama') // compile project(':trainbenchmark-tool-jastadd-specialized-base')
compile project(':trainbenchmark-tool-mysql') // compile project(':trainbenchmark-tool-jastadd-specialized-incremental')
compile project(':trainbenchmark-tool-neo4j') // compile project(':trainbenchmark-tool-jastadd-relast')
compile project(':trainbenchmark-tool-racr-python') // compile project(':trainbenchmark-tool-jastadd-relast-incremental')
compile project(':trainbenchmark-tool-racr-scheme')
compile project(':trainbenchmark-tool-racr-cpp')
compile project(':trainbenchmark-tool-rdf4j')
compile project(':trainbenchmark-tool-sqlite')
compile project(':trainbenchmark-tool-tinkergraph') compile project(':trainbenchmark-tool-tinkergraph')
compile project(':trainbenchmark-tool-viatra') compile project(':trainbenchmark-tool-viatra')
} }
......
...@@ -64,9 +64,13 @@ public class BenchmarkMain { ...@@ -64,9 +64,13 @@ public class BenchmarkMain {
ObjectMapper mapper = Utils.getMapper(); ObjectMapper mapper = Utils.getMapper();
BenchmarkSettingsBasics bbs = Utils.readFromResource(mapper, "basic-settings.json", BenchmarkSettingsBasics bbs = Utils.readFromResource(mapper, "basic-settings.json",
BenchmarkSettingsBasics.class); BenchmarkSettingsBasics.class);
try {
BenchmarkSettingsBasics loca_bbs = Utils.readFromResource(mapper, "local-basic-settings.json", BenchmarkSettingsBasics loca_bbs = Utils.readFromResource(mapper, "local-basic-settings.json",
BenchmarkSettingsBasics.class); BenchmarkSettingsBasics.class);
Utils.updateBasicsWithLocal(bbs, loca_bbs); Utils.updateBasicsWithLocal(bbs, loca_bbs);
} catch (IOException e) {
System.out.println("No file 'local-basic-settings.json' found. Using default!");
}
// import tools to use // import tools to use
List<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> builders = List<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> builders =
......
package de.tudresden.inf.st.train.scripts; package de.tudresden.inf.st.train.scripts;
import de.tudresden.inf.st.trainbenchmark.generator.json4ag.config.Json4AgGeneratorConfigBuilder; import de.tudresden.inf.st.trainbenchmark.generator.json4ag.config.Json4AgGeneratorConfigBuilder;
import de.tudresden.inf.st.trainbenchmark.generator.json4agref.config.Json4AgRefGeneratorConfigBuilder;
import hu.bme.mit.trainbenchmark.generator.config.*; import hu.bme.mit.trainbenchmark.generator.config.*;
import hu.bme.mit.trainbenchmark.generator.emf.config.EmfGeneratorConfigBuilder; import hu.bme.mit.trainbenchmark.generator.emf.config.EmfGeneratorConfigBuilder;
import hu.bme.mit.trainbenchmark.generator.graph.neo4j.config.Neo4jGraphGeneratorConfigBuilder;
import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphFormat; import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphFormat;
import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphGeneratorConfigBuilder; import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphGeneratorConfigBuilder;
import hu.bme.mit.trainbenchmark.generator.rdf.config.RdfGeneratorConfigBuilder;
import hu.bme.mit.trainbenchmark.generator.sql.config.SqlGeneratorConfigBuilder;
import hu.bme.mit.trainbenchmark.neo4j.config.Neo4jGraphFormat;
import hu.bme.mit.trainbenchmark.rdf.RdfFormat;
import java.util.*; import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
...@@ -45,15 +39,9 @@ public class BenchmarkSettingsAvailableFormats { ...@@ -45,15 +39,9 @@ public class BenchmarkSettingsAvailableFormats {
private static List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>> private static List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>>
getAllBuilders() { getAllBuilders() {
return Arrays.asList( return Arrays.asList(
new Neo4jGraphGeneratorConfigBuilder().setGraphFormat(Neo4jGraphFormat.CSV),
new Neo4jGraphGeneratorConfigBuilder().setGraphFormat(Neo4jGraphFormat.GRAPHML),
new EmfGeneratorConfigBuilder(), new EmfGeneratorConfigBuilder(),
new TinkerGraphGeneratorConfigBuilder().setGraphFormat(TinkerGraphFormat.GRAPHML), new TinkerGraphGeneratorConfigBuilder().setGraphFormat(TinkerGraphFormat.GRAPHML),
new Json4AgGeneratorConfigBuilder(), new Json4AgGeneratorConfigBuilder()
new Json4AgRefGeneratorConfigBuilder(),
new RdfGeneratorConfigBuilder().setFormat(RdfFormat.TURTLE).setInferred(true),
new RdfGeneratorConfigBuilder().setFormat(RdfFormat.TURTLE).setInferred(false),
new SqlGeneratorConfigBuilder()
); );
} }
......
package de.tudresden.inf.st.train.scripts; package de.tudresden.inf.st.train.scripts;
import de.tudresden.inf.st.train.kiama.config.KiamaBenchmarkConfigBuilder; import de.tudresden.inf.st.train.jastadd.config.JastaddNameLookupBenchmarkConfigBuilder;
import de.tudresden.inf.st.train.racr.config.RacrBenchmarkConfigBuilder; import de.tudresden.inf.st.train.jastadd.config.JastaddNameLookupIncrementalBenchmarkConfigBuilder;
import de.tudresden.inf.st.train.jastadd.config.JastaddOptimizedBenchmarkConfigBuilder;
import de.tudresden.inf.st.train.jastadd.config.JastaddOptimizedIncrementalBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.benchmark.config.*; import hu.bme.mit.trainbenchmark.benchmark.config.*;
import hu.bme.mit.trainbenchmark.benchmark.tinkergraph.config.TinkerGraphBenchmarkConfigBuilder; import hu.bme.mit.trainbenchmark.benchmark.tinkergraph.config.TinkerGraphBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.benchmark.viatra.config.ViatraBackend; import hu.bme.mit.trainbenchmark.benchmark.viatra.config.ViatraBackend;
import hu.bme.mit.trainbenchmark.benchmark.viatra.config.ViatraBenchmarkConfigBuilder; import hu.bme.mit.trainbenchmark.benchmark.viatra.config.ViatraBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.benchmark.drools.config.DroolsBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.benchmark.eclipseocl.config.EclipseOclBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.benchmark.emfapi.config.EmfApiBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.benchmark.jena.config.JenaBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.benchmark.mysql.config.MySqlBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.benchmark.sqlite.config.SQLiteBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.benchmark.neo4j.config.Neo4jBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.neo4j.config.Neo4jGraphFormat;
import hu.bme.mit.trainbenchmark.benchmark.neo4j.config.Neo4jEngine;
import hu.bme.mit.trainbenchmark.benchmark.rdf4j.config.Rdf4jBenchmarkConfigBuilder;
import java.util.*; import java.util.*;
import java.util.function.Function; import java.util.function.Function;
import java.util.stream.Collectors; import java.util.stream.Collectors;
...@@ -50,24 +41,10 @@ public class BenchmarkSettingsAvailableTools { ...@@ -50,24 +41,10 @@ public class BenchmarkSettingsAvailableTools {
private static List<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> private static List<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>>
getAllBuilders() { getAllBuilders() {
return Arrays.asList( return Arrays.asList(
new DroolsBenchmarkConfigBuilder(), new JastaddNameLookupBenchmarkConfigBuilder(),
new EclipseOclBenchmarkConfigBuilder(), new JastaddNameLookupIncrementalBenchmarkConfigBuilder(),
new EmfApiBenchmarkConfigBuilder(), new JastaddOptimizedBenchmarkConfigBuilder(),
getBuilderByName("de.tudresden.inf.st.train.jastadd.config.JastaddBenchmarkConfigBuilder"), new JastaddOptimizedIncrementalBenchmarkConfigBuilder(),
getBuilderByName("de.tudresden.inf.st.train.jastadd.config.JastaddIncrementalBenchmarkConfigBuilder"),
getBuilderByName("de.tudresden.inf.st.train.jastadd.config.JastaddSymbolicBenchmarkConfigBuilder"),
getBuilderByName("de.tudresden.inf.st.train.jastadd.config.JastaddSymbolicIncrementalBenchmarkConfigBuilder"),
new JenaBenchmarkConfigBuilder().setInferencing(false),
new JenaBenchmarkConfigBuilder().setInferencing(true),
new KiamaBenchmarkConfigBuilder(),
new MySqlBenchmarkConfigBuilder(),
new Neo4jBenchmarkConfigBuilder().setEngine(Neo4jEngine.COREAPI).setGraphFormat(Neo4jGraphFormat.CSV ),
new Neo4jBenchmarkConfigBuilder().setEngine(Neo4jEngine.CYPHER ).setGraphFormat(Neo4jGraphFormat.GRAPHML),
new RacrBenchmarkConfigBuilder().setVerbose(false).useCpp().useInternalServer(),
new RacrBenchmarkConfigBuilder().setVerbose(false).usePython(),
new RacrBenchmarkConfigBuilder().setVerbose(false).useScheme(),
new Rdf4jBenchmarkConfigBuilder().setInferencing(false),
new SQLiteBenchmarkConfigBuilder(),
new TinkerGraphBenchmarkConfigBuilder(), new TinkerGraphBenchmarkConfigBuilder(),
new ViatraBenchmarkConfigBuilder().setBackend(ViatraBackend.INCREMENTAL), new ViatraBenchmarkConfigBuilder().setBackend(ViatraBackend.INCREMENTAL),
new ViatraBenchmarkConfigBuilder().setBackend(ViatraBackend.LOCAL_SEARCH) new ViatraBenchmarkConfigBuilder().setBackend(ViatraBackend.LOCAL_SEARCH)
......
...@@ -5,6 +5,7 @@ import hu.bme.mit.trainbenchmark.config.ExecutionConfig; ...@@ -5,6 +5,7 @@ import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
import hu.bme.mit.trainbenchmark.generator.config.*; import hu.bme.mit.trainbenchmark.generator.config.*;
import hu.bme.mit.trainbenchmark.generator.runner.GeneratorRunner; import hu.bme.mit.trainbenchmark.generator.runner.GeneratorRunner;
import java.io.IOException;
import java.util.List; import java.util.List;
/** /**
...@@ -17,9 +18,13 @@ public class GenerateMain { ...@@ -17,9 +18,13 @@ public class GenerateMain {
ObjectMapper mapper = Utils.getMapper(); ObjectMapper mapper = Utils.getMapper();
BenchmarkSettingsBasics bbs = Utils.readFromResource(mapper, "basic-settings.json", BenchmarkSettingsBasics bbs = Utils.readFromResource(mapper, "basic-settings.json",
BenchmarkSettingsBasics.class); BenchmarkSettingsBasics.class);
try {
BenchmarkSettingsBasics loca_bbs = Utils.readFromResource(mapper, "local-basic-settings.json", BenchmarkSettingsBasics loca_bbs = Utils.readFromResource(mapper, "local-basic-settings.json",
BenchmarkSettingsBasics.class); BenchmarkSettingsBasics.class);
Utils.updateBasicsWithLocal(bbs, loca_bbs); Utils.updateBasicsWithLocal(bbs, loca_bbs);
} catch (IOException e) {
System.out.println("No file 'local-basic-settings.json' found. Using default!");
}
// import formats to use // import formats to use
List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>> builders = List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>> builders =
......
...@@ -5,24 +5,10 @@ ...@@ -5,24 +5,10 @@
"runs": 1, "runs": 1,
"dry-run": true, "dry-run": true,
"tools": [ "tools": [
"Drools", "Jastadd (Name Lookup)",
"Eclipse OCL", "Jastadd (Incremental Name Lookup)",
"EMF API", "Jastadd (Optimized)",
"Jena (Inferencing)", "Jastadd (Optimized Incremental)",
"Jena (No Inferencing)",
"MySQL",
"Neo4j (Core API-CSV)",
"Neo4j (Cypher-GraphML)",
"Racr (Scheme)",
"Racr (Python)",
"Racr (CPP)",
"RDF4J (No Inferencing)",
"SQLite",
"Jastadd (Symbolic References)",
"Jastadd (Incremental + Symbolic References)",
"Jastadd (Java References)",
"Jastadd (Incremental + Java References)",
"Kiama",
"TinkerGraph", "TinkerGraph",
"VIATRA (Incremental)", "VIATRA (Incremental)",
"VIATRA (Local Search)" "VIATRA (Local Search)"
...@@ -30,10 +16,7 @@ ...@@ -30,10 +16,7 @@
"formats": [ "formats": [
"emf", "emf",
"json4ag", "json4ag",
"json4ag-ref", "graph-tinkerpop"
"graph-tinkerpop",
"rdf",
"sql"
], ],
"jvmSettings": [ "jvmSettings": [
{ {
......
...@@ -4,10 +4,16 @@ public class ExecutionConfig { ...@@ -4,10 +4,16 @@ public class ExecutionConfig {
protected Integer initialMemory; protected Integer initialMemory;
protected Integer maxMemory; protected Integer maxMemory;
protected Boolean dryRun;
public ExecutionConfig(final Integer initialMemory, final Integer maxMemory) { public ExecutionConfig(final Integer initialMemory, final Integer maxMemory, final boolean dryRun) {
this.initialMemory = initialMemory; this.initialMemory = initialMemory;
this.maxMemory = maxMemory; this.maxMemory = maxMemory;
this.dryRun = dryRun;
}
public ExecutionConfig(final Integer initialMemory, final Integer maxMemory) {
this(initialMemory, maxMemory, true);
} }
/** /**
...@@ -42,6 +48,14 @@ public class ExecutionConfig { ...@@ -42,6 +48,14 @@ public class ExecutionConfig {
return maxMemory + "M"; return maxMemory + "M";
} }
/**
*
* @return whether this benchmark run must not start any Java processes
*/
public Boolean getDryRun() {
return dryRun;
}
public static ExecutionConfig defaultExecutionConfig() { public static ExecutionConfig defaultExecutionConfig() {
return new ExecutionConfig(1000, 1000); return new ExecutionConfig(1000, 1000);
} }
......
...@@ -24,7 +24,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase { ...@@ -24,7 +24,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase {
/** /**
* The id for the benchmark, used for determining the result directory. * The id for the benchmark, used for determining the result directory.
*/ */
protected final int benchmarkId; protected final String benchmarkId;
/** /**
* The timeout for each measurement run in seconds. * The timeout for each measurement run in seconds.
...@@ -68,7 +68,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase { ...@@ -68,7 +68,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase {
*/ */
protected final Optional<Integer> transformationConstant; protected final Optional<Integer> transformationConstant;
protected BenchmarkConfigBase(final int benchmarkId, final long timeout, final int runs, final String modelFilename, protected BenchmarkConfigBase(final String benchmarkId, final long timeout, final int runs, final String modelFilename,
final List<RailwayOperation> operations, final String workload, final List<RailwayOperation> operations, final String workload,
final TransformationChangeSetStrategy transformationChangeSetStrategy, final TransformationChangeSetStrategy transformationChangeSetStrategy,
final Optional<Integer> queryTransformationCount, final Optional<Integer> transformationConstant) { final Optional<Integer> queryTransformationCount, final Optional<Integer> transformationConstant) {
...@@ -84,7 +84,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase { ...@@ -84,7 +84,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase {
this.transformationConstant = transformationConstant; this.transformationConstant = transformationConstant;
} }
public int getBenchmarkId() { public String getBenchmarkId() {
return benchmarkId; return benchmarkId;
} }
......
...@@ -8,7 +8,7 @@ import com.google.common.base.Preconditions; ...@@ -8,7 +8,7 @@ import com.google.common.base.Preconditions;
import hu.bme.mit.trainbenchmark.constants.RailwayOperation; import hu.bme.mit.trainbenchmark.constants.RailwayOperation;
public final class BenchmarkConfigBaseBuilder { public final class BenchmarkConfigBaseBuilder {
private Integer benchmarkId; private String benchmarkId;
private Long timeout; private Long timeout;
private Integer runs; private Integer runs;
private String modelFilename; private String modelFilename;
...@@ -65,7 +65,7 @@ public final class BenchmarkConfigBaseBuilder { ...@@ -65,7 +65,7 @@ public final class BenchmarkConfigBaseBuilder {
return this; return this;
} }
public BenchmarkConfigBaseBuilder setBenchmarkId(final Integer benchmarkId) { public BenchmarkConfigBaseBuilder setBenchmarkId(final String benchmarkId) {
this.benchmarkId = benchmarkId; this.benchmarkId = benchmarkId;
return this; return this;
} }
......
...@@ -2,6 +2,7 @@ package hu.bme.mit.trainbenchmark.benchmark.result; ...@@ -2,6 +2,7 @@ package hu.bme.mit.trainbenchmark.benchmark.result;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.Charset;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
...@@ -24,8 +25,9 @@ public abstract class AbstractResult { ...@@ -24,8 +25,9 @@ public abstract class AbstractResult {
protected final String MODEL = "Model"; protected final String MODEL = "Model";
protected final String RUN = "Run"; protected final String RUN = "Run";
protected final int benchmarkId; protected final String benchmarkId;
protected final String toolName; protected final String toolName;
protected final String projectName;
protected final String workload; protected final String workload;
protected final String workspaceDir; protected final String workspaceDir;
protected final String model; protected final String model;
...@@ -34,6 +36,7 @@ public abstract class AbstractResult { ...@@ -34,6 +36,7 @@ public abstract class AbstractResult {
public AbstractResult(final BenchmarkConfig bc) { public AbstractResult(final BenchmarkConfig bc) {
this.benchmarkId = bc.getConfigBase().getBenchmarkId(); this.benchmarkId = bc.getConfigBase().getBenchmarkId();
this.toolName = bc.getToolName(); this.toolName = bc.getToolName();
this.projectName = bc.getProjectName();
this.workload = bc.getConfigBase().getWorkload(); this.workload = bc.getConfigBase().getWorkload();
this.workspaceDir = bc.getConfigBase().getWorkspaceDir(); this.workspaceDir = bc.getConfigBase().getWorkspaceDir();
this.model = bc.getConfigBase().getModelFilename(); this.model = bc.getConfigBase().getModelFilename();
...@@ -41,13 +44,15 @@ public abstract class AbstractResult { ...@@ -41,13 +44,15 @@ public abstract class AbstractResult {
} }
public void serializeCsv(final String csv, final String filePrefix) throws IOException { public void serializeCsv(final String csv, final String filePrefix) throws IOException {
final String matchesCsvPath = String.format("%s/%s-%s-%s-%s-%s.csv", getResultDir(), filePrefix, toolName, File targetDir = ResultHelper.getResultDirectory(projectName, benchmarkId);
workload, model, description); String fileName = String.format("%s-%s-%s-%s-%s.csv", filePrefix, toolName, workload, model, description);
FileUtils.write(new File(matchesCsvPath), csv); // final String matchesCsvPath = String.format("%s/%s-%s-%s-%s-%s.csv", getResultDir(), filePrefix, toolName,
// workload, model, description);
FileUtils.write(FileUtils.getFile(targetDir, fileName), csv, Charset.defaultCharset());
} }
public String getResultDir() { // public String getResultDir() {
return workspaceDir + ResultHelper.getResultDirForId(benchmarkId); // return workspaceDir + ResultHelper.getResultDirForId(benchmarkId);
} // }
} }
...@@ -3,8 +3,15 @@ package hu.bme.mit.trainbenchmark.benchmark.result; ...@@ -3,8 +3,15 @@ package hu.bme.mit.trainbenchmark.benchmark.result;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.file.*; import java.nio.file.*;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfig;
import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
...@@ -12,6 +19,7 @@ public final class ResultHelper { ...@@ -12,6 +19,7 @@ public final class ResultHelper {
public static final String WORKSPACE_DIR = "../"; public static final String WORKSPACE_DIR = "../";
public static final String RESULT_DIR = "results/"; public static final String RESULT_DIR = "results/";
public static final String TOOLS_DIR = "tools/";
public static final String DIAGRAM_DIR = "diagrams/"; public static final String DIAGRAM_DIR = "diagrams/";
public static final String SCRIPT_PATH = WORKSPACE_DIR + "trainbenchmark-scripts/src/"; public static final String SCRIPT_PATH = WORKSPACE_DIR + "trainbenchmark-scripts/src/";
public static final String BENCHMARK_SCRIPT = "BenchmarkScript.groovy"; public static final String BENCHMARK_SCRIPT = "BenchmarkScript.groovy";
...@@ -94,4 +102,38 @@ public final class ResultHelper { ...@@ -94,4 +102,38 @@ public final class ResultHelper {
FileUtils.copyFile(srcFile, destFile); FileUtils.copyFile(srcFile, destFile);
} }
public static void createNewResultDirs(Collection<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> tools) {
}
/**
* Prepare a benchmark run, i.e., creates necessary directories. This operation is idempotent.
* @param config the benchmark configuration
* @param ec the execution configuration
*/
public static void prepare(BenchmarkConfig config, ExecutionConfig ec) {
// ensure directory for the tool
File toolDir = getToolDirectory(config.getProjectName());
toolDir.mkdir();
// ensure directory for the run inside toolDir
File runDir = getResultDirectory(toolDir, config.getConfigBase().getBenchmarkId());
runDir.mkdir();
}
public static String getNow() {
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
return dateFormat.format(new Date());
}
public static File getToolDirectory(String projectName) {
return FileUtils.getFile(WORKSPACE_DIR, RESULT_DIR, TOOLS_DIR, projectName);
}
public static File getResultDirectory(File toolDirectory, String benchmarkId) {
return FileUtils.getFile(toolDirectory, "run-" + benchmarkId);
}
public static File getResultDirectory(String projectName, String benchmarkId) {
return getResultDirectory(getToolDirectory(projectName), benchmarkId);
}
} }
...@@ -11,14 +11,38 @@ import org.apache.commons.exec.Executor; ...@@ -11,14 +11,38 @@ import org.apache.commons.exec.Executor;
import org.apache.commons.exec.PumpStreamHandler; import org.apache.commons.exec.PumpStreamHandler;
import com.google.common.base.Joiner; import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableMap;
import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfig; import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfig;
import hu.bme.mit.trainbenchmark.benchmark.memory.MemoryResult; import hu.bme.mit.trainbenchmark.benchmark.memory.MemoryResult;
import hu.bme.mit.trainbenchmark.config.ExecutionConfig; import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
import org.apache.commons.exec.*;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Map;
public class BenchmarkRunner { public class BenchmarkRunner {
public static int runPerformanceBenchmark(final BenchmarkConfig bc, final ExecutionConfig ec) public static String dictFormat(String format, Map<String, Object> values) {
StringBuilder convFormat = new StringBuilder(format);
ArrayList<Object> valueList = new ArrayList<>();
int currentPos = 1;
for (String key : values.keySet()) {
String formatKey = "%(" + key + ")",
formatPos = "%" + Integer.toString(currentPos) + "$1s";
int index = -1;
while ((index = convFormat.indexOf(formatKey, index)) != -1) {
convFormat.replace(index, index + formatKey.length(), formatPos);
index += formatPos.length();
}
valueList.add(values.get(key));
++currentPos;
}
return String.format(convFormat.toString(), valueList.toArray());
}
public static int runPerformanceBenchmark(final BenchmarkConfig bc, final ExecutionConfig ec, String jvmSetting)
throws IOException, InterruptedException { throws IOException, InterruptedException {
final Joiner joiner = Joiner.on(", "); final Joiner joiner = Joiner.on(", ");
System.out.println("Running benchmark."); System.out.println("Running benchmark.");
...@@ -38,9 +62,17 @@ public class BenchmarkRunner { ...@@ -38,9 +62,17 @@ public class BenchmarkRunner {
final String jarPath = String.format("../%s/build/libs/%s-1.0.0-SNAPSHOT-fat.jar %s", projectName, projectName, final String jarPath = String.format("../%s/build/libs/%s-1.0.0-SNAPSHOT-fat.jar %s", projectName, projectName,
configPath); configPath);
final String javaCommand = String.format("java -Xms%s -Xmx%s -server -Xverify:none -jar %s %s", ec.getXms(), ec.getXmx(), jarPath, configPath); final String javaCommand = dictFormat("java " + jvmSetting + " -jar %(jarPath) %(configPath)",
ImmutableMap.of("Xms", ec.getXms(), "Xmx", ec.getXmx(),
"jarPath", jarPath, "configPath", configPath));
// final String javaCommand = String.format("java -Xms%s -Xmx%s -server -Xverify:none -jar %s %s", ec.getXms(), ec.getXmx(), jarPath, configPath);
// final String javaCommand = String.format("java -Xms%s -Xmx%s -server -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:StartFlightRecording=dumponexit=true -Xverify:none -jar %s %s", ec.getXms(), ec.getXmx(), jarPath, configPath); // final String javaCommand = String.format("java -Xms%s -Xmx%s -server -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:StartFlightRecording=dumponexit=true -Xverify:none -jar %s %s", ec.getXms(), ec.getXmx(), jarPath, configPath);
final CommandLine cmdLine = CommandLine.parse(javaCommand); final CommandLine cmdLine = CommandLine.parse(javaCommand);
if (ec.getDryRun()) {
System.out.println("Would have executed: "+ cmdLine.toString());
return 0;
}
final long timeoutInSeconds = bc.getConfigBase().getTimeout(); final long timeoutInSeconds = bc.getConfigBase().getTimeout();
final long timeoutInMilliseconds = timeoutInSeconds * 1000; final long timeoutInMilliseconds = timeoutInSeconds * 1000;
...@@ -63,7 +95,7 @@ public class BenchmarkRunner { ...@@ -63,7 +95,7 @@ public class BenchmarkRunner {
} }
public static int runMemoryBenchmark(final BenchmarkConfig bc, final ExecutionConfig defaultEc, public static int runMemoryBenchmark(final BenchmarkConfig bc, final ExecutionConfig defaultEc,
final int numberOfSteps) throws IOException, InterruptedException { final int numberOfSteps, String jvmSetting) throws IOException, InterruptedException {
// e.g. initialMaxMemory = 12800, we save this (as a final variable), so // e.g. initialMaxMemory = 12800, we save this (as a final variable), so
// that we will not exceed it // that we will not exceed it
// //
...@@ -79,7 +111,7 @@ public class BenchmarkRunner { ...@@ -79,7 +111,7 @@ public class BenchmarkRunner {
memoryQuantum /= 2; memoryQuantum /= 2;
final ExecutionConfig ec = new ExecutionConfig(currentMaxMemory, currentMaxMemory); final ExecutionConfig ec = new ExecutionConfig(currentMaxMemory, currentMaxMemory);
if (runPerformanceBenchmark(bc, ec) == 0) { if (runPerformanceBenchmark(bc, ec, jvmSetting) == 0) {
System.out.println("Execution finished, testing with less memory."); System.out.println("Execution finished, testing with less memory.");
System.out.println(); System.out.println();
currentMaxMemory -= memoryQuantum; currentMaxMemory -= memoryQuantum;
......
...@@ -23,7 +23,7 @@ public abstract class TrainBenchmarkTest { ...@@ -23,7 +23,7 @@ public abstract class TrainBenchmarkTest {
@Rule @Rule
public ErrorCollector collector = new ErrorCollector(); public ErrorCollector collector = new ErrorCollector();
protected final int benchmarkId = 0; protected final String benchmarkId = "";
protected ExecutionConfig executionConfig = ExecutionConfig.defaultExecutionConfig(); protected ExecutionConfig executionConfig = ExecutionConfig.defaultExecutionConfig();
protected final long timeout = 120; protected final long timeout = 120;
protected final int runs = 1; protected final int runs = 1;
......
...@@ -11,7 +11,7 @@ import hu.bme.mit.trainbenchmark.config.ExecutionConfig; ...@@ -11,7 +11,7 @@ import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
public abstract class QueryTest { public abstract class QueryTest {
protected final int benchmarkId = 0; protected final String benchmarkId = "";
protected ExecutionConfig executionConfig = ExecutionConfig.defaultExecutionConfig(); protected ExecutionConfig executionConfig = ExecutionConfig.defaultExecutionConfig();
protected final long timeout = 120; protected final long timeout = 120;
protected final int runs = 1; protected final int runs = 1;
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment