Skip to content
Snippets Groups Projects
Commit 85efe24f authored by René Schöne's avatar René Schöne
Browse files

Add alternate scripts.

parent b86e441a
Branches
No related tags found
No related merge requests found
Showing
with 1205 additions and 0 deletions
dependencies {
compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.2.3'
compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.3.0'
compile project(':trainbenchmark-generator')
compile project(':trainbenchmark-generator-emf')
compile project(':trainbenchmark-generator-graph-neo4j')
compile project(':trainbenchmark-generator-graph-tinkerpop')
compile project(':trainbenchmark-generator-rdf')
compile project(':trainbenchmark-generator-sql')
compile project(':trainbenchmark-generator-json4ag')
compile project(':trainbenchmark-generator-json4ag-ref')
compile project(':trainbenchmark-generator-dot')
compile project(':trainbenchmark-tool')
// compile project(':trainbenchmark-tool-blazegraph')
compile project(':trainbenchmark-tool-drools')
compile project(':trainbenchmark-tool-eclipseocl')
compile project(':trainbenchmark-tool-emfapi')
// compile project(':trainbenchmark-tool-epsilon')
// compile project(':trainbenchmark-tool-ingraph')
compile project(':trainbenchmark-tool-jastadd-base')
compile project(':trainbenchmark-tool-jastadd-symbolic-references')
compile project(':trainbenchmark-tool-jastadd-symbolic-references-base')
compile project(':trainbenchmark-tool-jastadd-symbolic-references-incremental')
compile project(':trainbenchmark-tool-jastadd-java-references')
compile project(':trainbenchmark-tool-jastadd-java-references-base')
compile project(':trainbenchmark-tool-jastadd-java-references-incremental')
compile project(':trainbenchmark-tool-jena')
compile project(':trainbenchmark-tool-kiama')
compile project(':trainbenchmark-tool-mysql')
compile project(':trainbenchmark-tool-neo4j')
compile project(':trainbenchmark-tool-racr-python')
compile project(':trainbenchmark-tool-racr-scheme')
compile project(':trainbenchmark-tool-racr-cpp')
compile project(':trainbenchmark-tool-rdf4j')
compile project(':trainbenchmark-tool-sqlite')
compile project(':trainbenchmark-tool-tinkergraph')
compile project(':trainbenchmark-tool-viatra')
}
task cleanResults(type: Delete) {
doLast {
def resultDir = "../results"
file(resultDir).eachDir{ dir ->
delete "${dir}"
}
}
}
task combinedBenchmark(dependsOn: 'classes', type: JavaExec) {
group = 'Benchmark'
description = 'Runs the \'classic\' TrainBenchmark script'
main = 'de.tudresden.inf.st.train.scripts.BenchmarkMain'
classpath = sourceSets.main.runtimeClasspath
args 'combined'
}
task individualRepairBenchmark(dependsOn: 'classes', type: JavaExec) {
group = 'Benchmark'
main = 'de.tudresden.inf.st.train.scripts.BenchmarkMain'
classpath = sourceSets.main.runtimeClasspath
args 'individual_repair'
}
task individualInjectBenchmark(dependsOn: 'classes', type: JavaExec) {
group = 'Benchmark'
main = 'de.tudresden.inf.st.train.scripts.BenchmarkMain'
classpath = sourceSets.main.runtimeClasspath
args 'individual_inject'
}
task individualIncrementalBenchmark(dependsOn: 'classes', type: JavaExec) {
group = 'Benchmark'
main = 'de.tudresden.inf.st.train.scripts.BenchmarkMain'
classpath = sourceSets.main.runtimeClasspath
args 'individual_incremental'
}
task generate(dependsOn: 'classes', type: JavaExec) {
group = 'Benchmark'
description = 'Generates the models for all benchmarks'
main = 'de.tudresden.inf.st.train.scripts.GenerateMain'
classpath = sourceSets.main.runtimeClasspath
}
//task memory(dependsOn: 'classes', type: JavaExec) {
// group = 'Benchmark'
// main = 'MemoryScript'
// classpath = sourceSets.main.runtimeClasspath
//}
//
//task qpt(dependsOn: 'classes', type: JavaExec) {
// group = 'Benchmark'
// main = 'QueryPlanTester'
// classpath = sourceSets.main.runtimeClasspath
//}
package de.tudresden.inf.st.train.scripts;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.ObjectMapper;
import hu.bme.mit.trainbenchmark.benchmark.config.*;
import hu.bme.mit.trainbenchmark.benchmark.result.ResultHelper;
import hu.bme.mit.trainbenchmark.benchmark.runcomponents.BenchmarkRunner;
import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
import hu.bme.mit.trainbenchmark.constants.RailwayOperation;
import java.io.IOException;
import java.util.*;
/**
* Main entry point to start benchmarks.
* Created by rschoene on 10/12/17.
*/
public class BenchmarkMain {
@SuppressWarnings("unchecked")
private static void runBenchmarkSeries(BenchmarkConfigBaseBuilder configBaseBuilder, BenchmarkConfigBuilder configBuilder,
ExecutionConfig ec, ModelSetConfig modelSetConfig, BenchmarkSettingsBasics bbs) {
Map<String, String> jvmSettingsMap = Utils.createJvmSettingsMap(bbs);
String jvmDefaultSettings = jvmSettingsMap.getOrDefault("*",
"-Xms{Xms} -Xmx{Xmx} -server -Xverify:none");
try {
for (int size = modelSetConfig.getMinSize(); size <= modelSetConfig.getMaxSize(); size *= 2) {
String modelFilename = "railway-" + modelSetConfig.getModelVariant() + "-" + size;
System.out.println("------------------------------------------------------------");
System.out.println("Model: " + modelFilename);
System.out.println("------------------------------------------------------------");
configBaseBuilder.setModelFilename(modelFilename);
BenchmarkConfigBase configBase = configBaseBuilder.createConfigBase();
BenchmarkConfig config = (BenchmarkConfig) configBuilder.setConfigBase(configBase).createConfig();
if (!bbs.isDryRun()) {
ResultHelper.prepare(config, ec);
}
String jvmSetting = jvmSettingsMap.getOrDefault(config.getToolName(), jvmDefaultSettings);
int exitValue = BenchmarkRunner.runPerformanceBenchmark(config, ec, jvmSetting);
if (exitValue != 0) {
System.out.println("Timeout or error occurred, skipping models for larger sizes. Error code: " + exitValue);
break;
}
}
} catch (InterruptedException | IOException e) {
System.out.println("Exception occurred during execution.");
e.printStackTrace();
}
}
public static void main(String args[]) throws Exception {
// set benchmark variant from program arguments
if (args.length == 0) {
System.err.println("Need to specify one argument identifying the benchmark variant");
System.exit(1);
}
String benchmarkVariant = args[0];
// read basic settings (min, maxsize, etc.)
ObjectMapper mapper = Utils.getMapper();
BenchmarkSettingsBasics bbs = Utils.readFromResource(mapper, "basic-settings.json",
BenchmarkSettingsBasics.class);
BenchmarkSettingsBasics loca_bbs = Utils.readFromResource(mapper, "local-basic-settings.json",
BenchmarkSettingsBasics.class);
Utils.updateBasicsWithLocal(bbs, loca_bbs);
// import tools to use
List<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> builders =
BenchmarkSettingsAvailableTools.getBuilders(bbs.getTools());
System.out.println(builders);
// read workloads based on benchmark variant
String workloadsFilename = "workloads.json";
BenchmarkSettingsVariantWorkloads bsvw = Utils.readFromResource(mapper, workloadsFilename,
BenchmarkSettingsVariantWorkloads.class);
Optional<BenchmarkSettingsWorkloads> workloadsOpt = bsvw.getVariants().stream()
.filter(w -> w.getName().equals(benchmarkVariant))
.findFirst();
if (!workloadsOpt.isPresent()) {
System.err.println("Could not find workloads for given variant, check " + workloadsFilename
+ " and argument " + benchmarkVariant);
System.exit(4);
}
BenchmarkSettingsWorkloads workloads = workloadsOpt.get();
// start benchmark
String now = ResultHelper.getNow();
ExecutionConfig ec = new ExecutionConfig(4000, 8000, bbs.isDryRun());
System.out.println("Please remember to stop all other Java processes.");
System.out.println();
System.out.println("If in doubt, check with this command:");
System.out.println("$ ps auxw | grep jav[a]");
System.out.println();
System.out.println("If there are other Java processes, use:");
System.out.println("$ killall -9 java");
System.out.println();
System.out.println("############################################################");
System.out.println("Benchmark parameters:");
System.out.println("- execution config: " + ec);
System.out.println(bbs);
// System.out.println("- range: minSize=" + bbs.getMinSize() +", maxSize=" + bbs.getMaxSize());
// System.out.println("- timeout: " + bbs.getTimeout());
// System.out.println("- runs: " + bbs.getRuns());
System.out.println("############################################################");
System.out.println();
workloads.getWorkloads().forEach(workloadConfiguration -> {
String workloadName = workloadConfiguration.getWorkloadName();
String modelVariant = workloadConfiguration.getModelVariant();
List<RailwayOperation> operations = workloadConfiguration.getOperations();
TransformationChangeSetStrategy strategy = workloadConfiguration.getStrategy();
int constant = workloadConfiguration.getConstant();
int queryTransformationCount = workloadConfiguration.getQueryTransformationCount();
System.out.println("============================================================");
System.out.println("Workload: " + workloadName);
System.out.println("============================================================");
ModelSetConfig modelSetConfig = new ModelSetConfig(modelVariant, bbs.getMinSize(), bbs.getMaxSize());
BenchmarkConfigBaseBuilder bcbb = new BenchmarkConfigBaseBuilder()
.setBenchmarkId(now).setTimeout(bbs.getTimeout()).setRuns(bbs.getRuns())
.setOperations(operations).setWorkload(workloadName)
.setQueryTransformationCount(queryTransformationCount).setTransformationConstant(constant)
.setTransformationChangeSetStrategy(strategy);
builders.forEach( bcb -> runBenchmarkSeries(bcbb, bcb, ec, modelSetConfig, bbs));
});
// if (binding.variables.get("reportUrl")) {
// BenchmarkReporter.reportReady(reportUrl)
// }
}
}
package de.tudresden.inf.st.train.scripts;
import de.tudresden.inf.st.trainbenchmark.generator.json4ag.config.Json4AgGeneratorConfigBuilder;
import de.tudresden.inf.st.trainbenchmark.generator.json4agref.config.Json4AgRefGeneratorConfigBuilder;
import hu.bme.mit.trainbenchmark.generator.config.*;
import hu.bme.mit.trainbenchmark.generator.emf.config.EmfGeneratorConfigBuilder;
import hu.bme.mit.trainbenchmark.generator.graph.neo4j.config.Neo4jGraphGeneratorConfigBuilder;
import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphFormat;
import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphGeneratorConfigBuilder;
import hu.bme.mit.trainbenchmark.generator.rdf.config.RdfGeneratorConfigBuilder;
import hu.bme.mit.trainbenchmark.generator.sql.config.SqlGeneratorConfigBuilder;
import hu.bme.mit.trainbenchmark.neo4j.config.Neo4jGraphFormat;
import hu.bme.mit.trainbenchmark.rdf.RdfFormat;
import java.util.*;
import java.util.stream.Collectors;
/**
* Format settings of a benchmark shared throughout all variants.
* Created by rschoene on 10/13/17.
*/
public class BenchmarkSettingsAvailableFormats {
/**
* Create a map, from projectName to a set of all builders in this project
* @return a mapping from projectName to all builders
*/
public static Map<String, List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>>>
getAvailableFormats() {
// { projectName -> [builder] }
GeneratorConfigBase dummyConfigBase = new GeneratorConfigBaseBuilder()
.setScenario(Scenario.BATCH).setSize(1)
.createGeneratorConfigBase();
return getAllBuilders().stream().collect(Collectors.toMap(
builder -> builder.setConfigBase(dummyConfigBase).createConfig().getProjectName(),
s -> new ArrayList<>(Collections.singletonList(s)),
(oldValue, newValue) -> { oldValue.addAll(newValue); return oldValue; }));
}
private static GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>
getBuilderByName(String className) {
Optional<Object> result = Utils.maybeCreateNewInstance(className);
return (GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>) result.orElse(null);
}
private static List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>>
getAllBuilders() {
return Arrays.asList(
new Neo4jGraphGeneratorConfigBuilder().setGraphFormat(Neo4jGraphFormat.CSV),
new Neo4jGraphGeneratorConfigBuilder().setGraphFormat(Neo4jGraphFormat.GRAPHML),
new EmfGeneratorConfigBuilder(),
new TinkerGraphGeneratorConfigBuilder().setGraphFormat(TinkerGraphFormat.GRAPHML),
new Json4AgGeneratorConfigBuilder(),
new Json4AgRefGeneratorConfigBuilder(),
new RdfGeneratorConfigBuilder().setFormat(RdfFormat.TURTLE).setInferred(true),
new RdfGeneratorConfigBuilder().setFormat(RdfFormat.TURTLE).setInferred(false),
new SqlGeneratorConfigBuilder()
);
}
/**
* Get all builders with the given names of formats. The format name in the config every builder creates
* must match one of the given project names.
* @param formatNames names of formats
* @return a set of matching generator config builders
*/
public static List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>>
getBuilders(String ... formatNames) {
Map<String, List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>>>
availableFormats = getAvailableFormats();
return Arrays.stream(formatNames).map(name -> {
List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>>
builders = availableFormats.get(name);
if (builders.isEmpty()) {
throw new IllegalArgumentException("Could not find format " + name + ", available formats are: "
+ availableFormats.keySet());
}
return builders;
}).reduce((set1, set2) -> {
set1.addAll(set2);
return set1;
}).orElse(Collections.emptyList());
}
}
package de.tudresden.inf.st.train.scripts;
import de.tudresden.inf.st.train.kiama.config.KiamaBenchmarkConfigBuilder;
import de.tudresden.inf.st.train.racr.config.RacrBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.benchmark.config.*;
import hu.bme.mit.trainbenchmark.benchmark.tinkergraph.config.TinkerGraphBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.benchmark.viatra.config.ViatraBackend;
import hu.bme.mit.trainbenchmark.benchmark.viatra.config.ViatraBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.benchmark.drools.config.DroolsBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.benchmark.eclipseocl.config.EclipseOclBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.benchmark.emfapi.config.EmfApiBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.benchmark.jena.config.JenaBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.benchmark.mysql.config.MySqlBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.benchmark.sqlite.config.SQLiteBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.benchmark.neo4j.config.Neo4jBenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.neo4j.config.Neo4jGraphFormat;
import hu.bme.mit.trainbenchmark.benchmark.neo4j.config.Neo4jEngine;
import hu.bme.mit.trainbenchmark.benchmark.rdf4j.config.Rdf4jBenchmarkConfigBuilder;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* Tool settings of a benchmark shared throughout all variants
* Created by rschoene on 10/12/17.
*/
public class BenchmarkSettingsAvailableTools {
public static Map<String, BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>>
getAvailableTools() {
// { toolName -> builder }
BenchmarkConfigBase dummyConfigBase = new BenchmarkConfigBaseBuilder()
.setBenchmarkId("").setTimeout(0L).setRuns(0).setModelFilename("")
.setOperations(new ArrayList<>()).setWorkload("")
.setTransformationChangeSetStrategy(TransformationChangeSetStrategy.FIXED)
.setQueryTransformationCount(0).setTransformationConstant(0)
.createConfigBase();
return getAllBuilders().stream().collect(Collectors.toMap(
builder -> builder.setConfigBase(dummyConfigBase).createConfig().getToolName(),
Function.identity()));
}
private static BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>
getBuilderByName(String className) {
Optional<Object> result = Utils.maybeCreateNewInstance(className);
return (BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>) result.orElse(null);
}
private static List<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>>
getAllBuilders() {
return Arrays.asList(
new DroolsBenchmarkConfigBuilder(),
new EclipseOclBenchmarkConfigBuilder(),
new EmfApiBenchmarkConfigBuilder(),
getBuilderByName("de.tudresden.inf.st.train.jastadd.config.JastaddBenchmarkConfigBuilder"),
getBuilderByName("de.tudresden.inf.st.train.jastadd.config.JastaddIncrementalBenchmarkConfigBuilder"),
getBuilderByName("de.tudresden.inf.st.train.jastadd.config.JastaddSymbolicBenchmarkConfigBuilder"),
getBuilderByName("de.tudresden.inf.st.train.jastadd.config.JastaddSymbolicIncrementalBenchmarkConfigBuilder"),
new JenaBenchmarkConfigBuilder().setInferencing(false),
new JenaBenchmarkConfigBuilder().setInferencing(true),
new KiamaBenchmarkConfigBuilder(),
new MySqlBenchmarkConfigBuilder(),
new Neo4jBenchmarkConfigBuilder().setEngine(Neo4jEngine.COREAPI).setGraphFormat(Neo4jGraphFormat.CSV ),
new Neo4jBenchmarkConfigBuilder().setEngine(Neo4jEngine.CYPHER ).setGraphFormat(Neo4jGraphFormat.GRAPHML),
new RacrBenchmarkConfigBuilder().setVerbose(false).useCpp().useInternalServer(),
new RacrBenchmarkConfigBuilder().setVerbose(false).usePython(),
new RacrBenchmarkConfigBuilder().setVerbose(false).useScheme(),
new Rdf4jBenchmarkConfigBuilder().setInferencing(false),
new SQLiteBenchmarkConfigBuilder(),
new TinkerGraphBenchmarkConfigBuilder(),
new ViatraBenchmarkConfigBuilder().setBackend(ViatraBackend.INCREMENTAL),
new ViatraBenchmarkConfigBuilder().setBackend(ViatraBackend.LOCAL_SEARCH)
);
}
/**
* Get all builders with the given names of tools. The tool name in the config every builder creates must match one
* of the given tool names.
* @param toolNames names of tools
* @return a list of matching benchmark config builders
*/
public static List<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>>
getBuilders(String ... toolNames) {
Map<String, BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> availableTools = getAvailableTools();
return Arrays.stream(toolNames).map(name -> {
BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>> builder = availableTools.get(name);
if (builder == null) {
throw new IllegalArgumentException("Could not find tool " + name + ", available tools are: " + availableTools.keySet());
}
return builder;
}).collect(Collectors.toList());
}
}
package de.tudresden.inf.st.train.scripts;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonSetter;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
/**
* Basic settings bean of a benchmark shared throughout all variants.
* Created by rschoene on 10/12/17.
*/
public class BenchmarkSettingsBasics {
static class JVMSetting {
private List<String> applyTo;
private String args;
public List<String> getApplyTo() {
return applyTo;
}
public void setApplyTo(List<String> applyTo) {
this.applyTo = applyTo;
}
public String getArgs() {
return args;
}
public void setArgs(String args) {
this.args = args;
}
}
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
private Integer minSize = null;
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
private Integer maxSize = null;
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
private Long timeout = null;
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
private Integer runs = null;
@JsonInclude(JsonInclude.Include.NON_DEFAULT)
private Boolean dryRun = null;
private String[] tools;
private String[] formats;
private List<JVMSetting> jvmSettings;
/** Minimum model size */
public Integer getMinSize() {
return minSize;
}
public void setMinSize(int minSize) {
this.minSize = minSize;
}
/** Maximum model size */
public Integer getMaxSize() {
return maxSize;
}
public void setMaxSize(int maxSize) {
this.maxSize = maxSize;
}
/** Timeout in seconds for each single run. */
public Long getTimeout() {
return timeout;
}
public void setTimeout(long timeout) {
this.timeout = timeout;
}
/** Number of runs per tool-workload combination. */
public Integer getRuns() {
return runs;
}
public void setRuns(int runs) {
this.runs = runs;
}
/** Don't run any Java processes. */
public Boolean isDryRun() {
return dryRun;
}
@JsonSetter("dry-run")
public void setDryRun(boolean dryRun) {
this.dryRun = dryRun;
}
/** List of tool names to use */
public String[] getTools() {
return tools;
}
public void setTools(String[] tools) {
this.tools = tools;
}
/** List of format names to use */
public String[] getFormats() {
return formats;
}
public void setFormats(String[] formats) {
this.formats = formats;
}
public List<JVMSetting> getJvmSettings() {
return jvmSettings;
}
public void setJvmSettings(List<JVMSetting> jvmSettings) {
this.jvmSettings = jvmSettings;
}
@Override
public String toString() {
return "BenchmarkSettingsBasics{" +
"minSize=" + minSize +
", maxSize=" + maxSize +
", timeout=" + timeout +
", runs=" + runs +
", dryRun=" + dryRun +
", tools=" + Arrays.toString(tools) +
'}';
}
}
package de.tudresden.inf.st.train.scripts;
import java.util.List;
/**
* List of all workload settings by variant.
* Created by rschoene on 10/12/17.
*/
public class BenchmarkSettingsVariantWorkloads {
private List<BenchmarkSettingsWorkloads> variants;
public List<BenchmarkSettingsWorkloads> getVariants() {
return variants;
}
public void setVariants(List<BenchmarkSettingsWorkloads> variants) {
this.variants = variants;
}
}
package de.tudresden.inf.st.train.scripts;
import com.fasterxml.jackson.annotation.JsonIgnore;
import hu.bme.mit.trainbenchmark.benchmark.config.TransformationChangeSetStrategy;
import hu.bme.mit.trainbenchmark.constants.RailwayOperation;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* Benchmark setting workload bean.
* Created by rschoene on 10/12/17.
*/
public class BenchmarkSettingsWorkload {
private String workloadName;
private String modelVariant;
private List<RailwayOperation> operations;
private TransformationChangeSetStrategy strategy;
// private String strategy;
private int constant;
private int queryTransformationCount;
@JsonIgnore
private Map<String, RailwayOperation> railwayOps;
public BenchmarkSettingsWorkload() {
railwayOps = Arrays.stream(RailwayOperation.values()).collect(Collectors.toMap(
RailwayOperation::toString, Function.identity()
));
}
public String getWorkloadName() {
return workloadName;
}
public void setWorkloadName(String workloadName) {
this.workloadName = workloadName;
}
public String getModelVariant() {
return modelVariant;
}
public void setModelVariant(String modelVariant) {
this.modelVariant = modelVariant;
}
public List<RailwayOperation> getOperations() {
return operations;
}
public void setOperations(List<String> operations) {
this.operations = operations.stream().map(op -> railwayOps.get(op)).collect(Collectors.toList());
}
public TransformationChangeSetStrategy getStrategy() {
return strategy;
}
public void setStrategy(TransformationChangeSetStrategy strategy) {
this.strategy = strategy;
}
public int getConstant() {
return constant;
}
public void setConstant(int constant) {
this.constant = constant;
}
public int getQueryTransformationCount() {
return queryTransformationCount;
}
public void setQueryTransformationCount(int queryTransformationCount) {
this.queryTransformationCount = queryTransformationCount;
}
}
package de.tudresden.inf.st.train.scripts;
import java.util.List;
/**
* Workload settings bean.
* Created by rschoene on 10/12/17.
*/
public class BenchmarkSettingsWorkloads
{
private String name;
private List<BenchmarkSettingsWorkload> workloads;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public List<BenchmarkSettingsWorkload> getWorkloads() {
return workloads;
}
public void setWorkloads(List<BenchmarkSettingsWorkload> workloads) {
this.workloads = workloads;
}
}
package de.tudresden.inf.st.train.scripts;
/**
* Possible variants to run a benchmark in.
* Created by rschoene on 10/12/17.
*/
public enum BenchmarkVariant {
COMBINED,
INDIVIDUAL_INJECT,
INDIVIDUAL_REPAIR,
INDIVIDUAL_INCREMENTAL
}
package de.tudresden.inf.st.train.scripts;
import com.fasterxml.jackson.databind.ObjectMapper;
import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
import hu.bme.mit.trainbenchmark.generator.config.*;
import hu.bme.mit.trainbenchmark.generator.runner.GeneratorRunner;
import java.util.List;
/**
* Main entry point to generate models.
* Created by rschoene on 10/13/17.
*/
public class GenerateMain {
public static void main(String[] args) throws Exception {
// read basic settings (min, maxsize, etc.)
ObjectMapper mapper = Utils.getMapper();
BenchmarkSettingsBasics bbs = Utils.readFromResource(mapper, "basic-settings.json",
BenchmarkSettingsBasics.class);
BenchmarkSettingsBasics loca_bbs = Utils.readFromResource(mapper, "local-basic-settings.json",
BenchmarkSettingsBasics.class);
Utils.updateBasicsWithLocal(bbs, loca_bbs);
// import formats to use
List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>> builders =
BenchmarkSettingsAvailableFormats.getBuilders(bbs.getFormats());
System.out.println(builders);
ExecutionConfig ec = new ExecutionConfig(4000, 8000, bbs.isDryRun());
// start generating (exclude Scenario.MINIMAL)
for (Scenario scenario : new Scenario[]{Scenario.BATCH, Scenario.INJECT, Scenario.REPAIR}) {
builders.forEach( generatorConfigBuilder -> {
try {
for (int size = bbs.getMinSize(); size <= bbs.getMaxSize(); size *= 2) {
System.out.println("Scenario: " + scenario + ", size: " + size);
GeneratorConfigBase configBase = new GeneratorConfigBaseBuilder()
.setSize(size).setScenario(scenario)
.createGeneratorConfigBase();
GeneratorConfig config = generatorConfigBuilder.setConfigBase(configBase).createConfig();
int exitValue = GeneratorRunner.run(config, ec);
if (exitValue != 0) {
System.out.println("Timeout or error occurred, skipping models for larger sizes. Error code: "
+ exitValue);
break;
}
}
} catch (Exception e) {
System.out.println("Exception occurred during execution.");
}
});
}
}
}
/*
import de.tudresden.inf.st.trainbenchmark.generator.json4ag.config.Json4AgGeneratorConfigBuilder
import de.tudresden.inf.st.trainbenchmark.generator.json4agref.config.Json4AgRefGeneratorConfigBuilder
import hu.bme.mit.trainbenchmark.config.ExecutionConfig
import hu.bme.mit.trainbenchmark.generator.config.GeneratorConfigBase
import hu.bme.mit.trainbenchmark.generator.config.Scenario
import hu.bme.mit.trainbenchmark.generator.emf.config.EmfGeneratorConfigBuilder
import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphFormat
import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphGeneratorConfigBuilder
import hu.bme.mit.trainbenchmark.generator.runner.GeneratorRunner
def ec = new ExecutionConfig(4000, 6000)
def minSize = 1
/* def maxSize = 2048 *
def maxSize = 256
def scenarios = [
Scenario.BATCH,
Scenario.INJECT,
Scenario.REPAIR,
]
def formats = [
new EmfGeneratorConfigBuilder(),
new TinkerGraphGeneratorConfigBuilder().setGraphFormat(TinkerGraphFormat.GRAPHML),
new Json4AgGeneratorConfigBuilder(),
new Json4AgRefGeneratorConfigBuilder()
]
for (scenario in scenarios) {
formats.each { generatorConfigBuilder ->
try {
for (def size = minSize; size <= maxSize; size *= 2) {
println("Scenario: ${scenario}, size: ${size}")
def configBase = new GeneratorConfigBase(scenario, size)
def config = generatorConfigBuilder.setConfigBase(configBase).createConfig()
def exitValue = GeneratorRunner.run(config, ec)
if (exitValue != 0) {
println "Timeout or error occured, skipping models for larger sizes. Error code: ${exitValue}"
break
}
}
} catch (all) {
println "Exception occured during execution."
}
}
}
*/
package de.tudresden.inf.st.train.scripts;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.URL;
import java.util.*;
import java.util.stream.Collectors;
/**
* Utility methods for the main entry points.
* Created by rschoene on 10/13/17.
*/
public class Utils {
static Map<String, String> createJvmSettingsMap(BenchmarkSettingsBasics bbs) {
Map<String, String> result = new HashMap<>();
for (BenchmarkSettingsBasics.JVMSetting setting : bbs.getJvmSettings()) {
for (String toolName : setting.getApplyTo()) {
result.put(toolName, setting.getArgs());
}
}
return result;
}
private static File readFromResource(String filename) throws IOException {
URL basicSettingsURL = BenchmarkMain.class.getClassLoader().getResource(filename);
if (basicSettingsURL == null) {
System.err.println();
throw new IOException("Could not access " + filename + ". Exiting.");
}
return new File(basicSettingsURL.getFile());
}
static <T> T readFromResource(ObjectMapper mapper, String filename, Class<T> clazz) throws IOException {
File basicSettingsFile = readFromResource(filename);
T result = null;
try {
result = mapper.readValue(basicSettingsFile, clazz);
} catch (Exception e) {
System.err.println("Could not load '" + filename + "'. Exiting.");
e.printStackTrace();
System.exit(2);
}
return result;
}
static void updateBasicsWithLocal(BenchmarkSettingsBasics base, BenchmarkSettingsBasics local)
throws IllegalStateException {
List<Method> setter = Arrays.stream(BenchmarkSettingsBasics.class.getMethods())
.filter(m -> m.getName().startsWith("set"))
.collect(Collectors.toList());
for (Method method : BenchmarkSettingsBasics.class.getMethods()) {
final String name = method.getName().startsWith("get") ?
method.getName().substring(3) :
( method.getName().startsWith("is") ? method.getName().substring(2) : null);
if (name == null) {
continue;
}
try {
Object result = method.invoke(local);
if (result != null) {
Optional<Method> matchingSetter = setter.stream()
.filter(m -> m.getName().endsWith(name))
.findFirst();
if (matchingSetter.isPresent()) {
matchingSetter.get().invoke(base, result);
}
}
// now do sanity check if set in updated base
result = method.invoke(base);
if (result == null) {
throw new IllegalStateException("Basic setting for " + name + " is not given.");
}
} catch (IllegalAccessException | InvocationTargetException e) {
e.printStackTrace();
}
}
}
static ObjectMapper getMapper() {
ObjectMapper mapper = new ObjectMapper();
mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true);
return mapper;
}
static Optional<Object> maybeCreateNewInstance(final String className) {
try {
return Optional.of(Class.forName(className).newInstance());
} catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
System.err.println("Could not find builder class '" + className + "'");
e.printStackTrace();
return Optional.empty();
}
}
}
# basic settings can still be commited using --force
local-basic-settings.json
{
"minSize": 1,
"maxSize": 8,
"timeout": 900,
"runs": 1,
"dry-run": true,
"tools": [
"Drools",
"Eclipse OCL",
"EMF API",
"Jena (Inferencing)",
"Jena (No Inferencing)",
"MySQL",
"Neo4j (Core API-CSV)",
"Neo4j (Cypher-GraphML)",
"Racr (Scheme)",
"Racr (Python)",
"Racr (CPP)",
"RDF4J (No Inferencing)",
"SQLite",
"Jastadd (Symbolic References)",
"Jastadd (Incremental + Symbolic References)",
"Jastadd (Java References)",
"Jastadd (Incremental + Java References)",
"Kiama",
"TinkerGraph",
"VIATRA (Incremental)",
"VIATRA (Local Search)"
],
"formats": [
"emf",
"json4ag",
"json4ag-ref",
"graph-tinkerpop",
"rdf",
"sql"
],
"jvmSettings": [
{
"applyTo": ["*"],
"args": "-Xms%(Xms) -Xmx%(Xmx) -server -Xverify:none"
// "args": "-Xms%(Xms) -Xmx%(Xmx) -server -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:StartFlightRecording=dumponexit=true -Xverify:none"
},
{
"applyTo": ["Racr (CPP)", "Racr (Python)", "Racr (Scheme)"],
"args": "-Xms50M -Xmx50M -server -Xverify:none"
}
]
}
{
"variants": [
{
"name": "individual_repair",
"workloads": [
{
"workloadName": "ConnectedSegments",
"operations": [
"ConnectedSegmentsRepair"
],
"constant": 5,
"queryTransformationCount": 8,
"modelVariant": "repair",
"strategy": "PROPORTIONAL"
},
{
"workloadName": "PosLength",
"operations": [
"PosLengthRepair"
],
"constant": 5,
"queryTransformationCount": 8,
"modelVariant": "repair",
"strategy": "PROPORTIONAL"
},
{
"workloadName": "RouteSensor",
"operations": [
"RouteSensorRepair"
],
"constant": 5,
"queryTransformationCount": 8,
"modelVariant": "repair",
"strategy": "PROPORTIONAL"
},
{
"workloadName": "SemaphoreNeighbor",
"operations": [
"SemaphoreNeighborRepair"
],
"constant": 5,
"queryTransformationCount": 8,
"modelVariant": "repair",
"strategy": "PROPORTIONAL"
},
{
"workloadName": "SwitchMonitored",
"operations": [
"SwitchMonitoredRepair"
],
"constant": 5,
"queryTransformationCount": 8,
"modelVariant": "repair",
"strategy": "PROPORTIONAL"
},
{
"workloadName": "SwitchSet",
"operations": [
"SwitchSetRepair"
],
"constant": 5,
"queryTransformationCount": 8,
"modelVariant": "repair",
"strategy": "PROPORTIONAL"
}
]
},
{
"name": "combined",
"workloads": [
{
"workloadName": "Repair",
"operations": [
"ConnectedSegmentsRepair",
"PosLengthRepair",
"RouteSensorRepair",
"SemaphoreNeighborRepair",
"SwitchMonitoredRepair",
"SwitchSetRepair"
],
"constant": 5,
"queryTransformationCount": 8,
"modelVariant": "repair",
"strategy": "PROPORTIONAL"
},
{
"workloadName": "Inject",
"operations": [
"ConnectedSegments",
"PosLength",
"RouteSensor",
"SemaphoreNeighbor",
"SwitchMonitored",
"SwitchSet",
"ConnectedSegmentsInject",
"PosLengthInject",
"RouteSensorInject",
"SemaphoreNeighborInject",
"SwitchMonitoredInject",
"SwitchSetInject"
],
"constant": 10,
"queryTransformationCount": 12,
"modelVariant": "inject",
"strategy": "FIXED"
}
]
},
{
"name": "individual_inject",
"workloads": [
{
"workloadName": "ConnectedSegments",
"operations": [
"ConnectedSegments",
"ConnectedSegmentsInject"
],
"constant": 10,
"queryTransformationCount": 12,
"modelVariant": "inject",
"strategy": "FIXED"
},
{
"workloadName": "PosLength",
"operations": [
"PosLength",
"PosLengthInject"
],
"constant": 10,
"queryTransformationCount": 12,
"modelVariant": "inject",
"strategy": "FIXED"
},
{
"workloadName": "RouteSensor",
"operations": [
"RouteSensor",
"RouteSensorInject"
],
"constant": 10,
"queryTransformationCount": 12,
"modelVariant": "inject",
"strategy": "FIXED"
},
{
"workloadName": "SemaphoreNeighbor",
"operations": [
"SemaphoreNeighbor",
"SemaphoreNeighborInject"
],
"constant": 10,
"queryTransformationCount": 12,
"modelVariant": "inject",
"strategy": "FIXED"
},
{
"workloadName": "SwitchMonitored",
"operations": [
"SwitchMonitored",
"SwitchMonitoredInject"
],
"constant": 10,
"queryTransformationCount": 12,
"modelVariant": "inject",
"strategy": "FIXED"
},
{
"workloadName": "SwitchSet",
"operations": [
"SwitchSet",
"SwitchSetInject"
],
"constant": 10,
"queryTransformationCount": 12,
"modelVariant": "inject",
"strategy": "FIXED"
}
]
},
{
"name": "individual_incremental",
"workloads": [
{
"workloadName": "ConnectedSegments",
"operations": [
"ConnectedSegments",
"ConnectedSegmentsInject",
"ConnectedSegmentsRepair"
],
"constant": 10,
"queryTransformationCount": 50,
"modelVariant": "inject",
"strategy": "FIXED"
},
{
"workloadName": "PosLength",
"operations": [
"PosLength",
"PosLengthInject",
"PosLengthRepair"
],
"constant": 10,
"queryTransformationCount": 50,
"modelVariant": "inject",
"strategy": "FIXED"
},
{
"workloadName": "RouteSensor",
"operations": [
"RouteSensor",
"RouteSensorInject",
"RouteSensorRepair"
],
"constant": 10,
"queryTransformationCount": 50,
"modelVariant": "inject",
"strategy": "FIXED"
},
{
"workloadName": "SemaphoreNeighbor",
"operations": [
"SemaphoreNeighbor",
"SemaphoreNeighborInject",
"SemaphoreNeighborRepair"
],
"constant": 10,
"queryTransformationCount": 50,
"modelVariant": "inject",
"strategy": "FIXED"
},
{
"workloadName": "SwitchMonitored",
"operations": [
"SwitchMonitored",
"SwitchMonitoredInject",
"SwitchMonitoredRepair"
],
"constant": 10,
"queryTransformationCount": 50,
"modelVariant": "inject",
"strategy": "FIXED"
},
{
"workloadName": "SwitchSet",
"operations": [
"SwitchSet",
"SwitchSetInject",
"SwitchSetRepair"
],
"constant": 10,
"queryTransformationCount": 50,
"modelVariant": "inject",
"strategy": "FIXED"
}
]
}
]
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment