From 85efe24f9c9be69c21c5a781360fb02acc5cabba Mon Sep 17 00:00:00 2001 From: rschoene <rene.schoene@tu-dresden.de> Date: Fri, 15 Mar 2019 19:27:40 +0100 Subject: [PATCH] Add alternate scripts. --- .../build.gradle | 97 +++++++ .../inf/st/train/scripts/BenchmarkMain.java | 140 ++++++++++ .../BenchmarkSettingsAvailableFormats.java | 83 ++++++ .../BenchmarkSettingsAvailableTools.java | 94 +++++++ .../scripts/BenchmarkSettingsBasics.java | 134 +++++++++ .../BenchmarkSettingsVariantWorkloads.java | 19 ++ .../scripts/BenchmarkSettingsWorkload.java | 81 ++++++ .../scripts/BenchmarkSettingsWorkloads.java | 29 ++ .../st/train/scripts/BenchmarkVariant.java | 12 + .../inf/st/train/scripts/GenerateMain.java | 108 ++++++++ .../tudresden/inf/st/train/scripts/Utils.java | 99 +++++++ .../src/main/resources/.gitignore | 2 + .../src/main/resources/basic-settings.json | 49 ++++ .../src/main/resources/workloads.json | 258 ++++++++++++++++++ 14 files changed, 1205 insertions(+) create mode 100644 trainbenchmark/trainbenchmark-alternate-scripts/build.gradle create mode 100644 trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkMain.java create mode 100644 trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableFormats.java create mode 100644 trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableTools.java create mode 100644 trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsBasics.java create mode 100644 trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsVariantWorkloads.java create mode 100644 trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsWorkload.java create mode 100644 trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsWorkloads.java create mode 100644 trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkVariant.java create mode 100644 trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/GenerateMain.java create mode 100644 trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/Utils.java create mode 100644 trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/.gitignore create mode 100644 trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/basic-settings.json create mode 100644 trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/workloads.json diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/build.gradle b/trainbenchmark/trainbenchmark-alternate-scripts/build.gradle new file mode 100644 index 000000000..295ef68a2 --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/build.gradle @@ -0,0 +1,97 @@ +dependencies { + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.2.3' + compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.3.0' + + compile project(':trainbenchmark-generator') + compile project(':trainbenchmark-generator-emf') + compile project(':trainbenchmark-generator-graph-neo4j') + compile project(':trainbenchmark-generator-graph-tinkerpop') + compile project(':trainbenchmark-generator-rdf') + compile project(':trainbenchmark-generator-sql') + compile project(':trainbenchmark-generator-json4ag') + compile project(':trainbenchmark-generator-json4ag-ref') + compile project(':trainbenchmark-generator-dot') + compile project(':trainbenchmark-tool') +// compile project(':trainbenchmark-tool-blazegraph') + compile project(':trainbenchmark-tool-drools') + compile project(':trainbenchmark-tool-eclipseocl') + compile project(':trainbenchmark-tool-emfapi') +// compile project(':trainbenchmark-tool-epsilon') +// compile project(':trainbenchmark-tool-ingraph') + compile project(':trainbenchmark-tool-jastadd-base') + compile project(':trainbenchmark-tool-jastadd-symbolic-references') + compile project(':trainbenchmark-tool-jastadd-symbolic-references-base') + compile project(':trainbenchmark-tool-jastadd-symbolic-references-incremental') + compile project(':trainbenchmark-tool-jastadd-java-references') + compile project(':trainbenchmark-tool-jastadd-java-references-base') + compile project(':trainbenchmark-tool-jastadd-java-references-incremental') + compile project(':trainbenchmark-tool-jena') + compile project(':trainbenchmark-tool-kiama') + compile project(':trainbenchmark-tool-mysql') + compile project(':trainbenchmark-tool-neo4j') + compile project(':trainbenchmark-tool-racr-python') + compile project(':trainbenchmark-tool-racr-scheme') + compile project(':trainbenchmark-tool-racr-cpp') + compile project(':trainbenchmark-tool-rdf4j') + compile project(':trainbenchmark-tool-sqlite') + compile project(':trainbenchmark-tool-tinkergraph') + compile project(':trainbenchmark-tool-viatra') +} + +task cleanResults(type: Delete) { + doLast { + def resultDir = "../results" + file(resultDir).eachDir{ dir -> + delete "${dir}" + } + } +} + +task combinedBenchmark(dependsOn: 'classes', type: JavaExec) { + group = 'Benchmark' + description = 'Runs the \'classic\' TrainBenchmark script' + main = 'de.tudresden.inf.st.train.scripts.BenchmarkMain' + classpath = sourceSets.main.runtimeClasspath + args 'combined' +} + +task individualRepairBenchmark(dependsOn: 'classes', type: JavaExec) { + group = 'Benchmark' + main = 'de.tudresden.inf.st.train.scripts.BenchmarkMain' + classpath = sourceSets.main.runtimeClasspath + args 'individual_repair' +} + +task individualInjectBenchmark(dependsOn: 'classes', type: JavaExec) { + group = 'Benchmark' + main = 'de.tudresden.inf.st.train.scripts.BenchmarkMain' + classpath = sourceSets.main.runtimeClasspath + args 'individual_inject' +} + +task individualIncrementalBenchmark(dependsOn: 'classes', type: JavaExec) { + group = 'Benchmark' + main = 'de.tudresden.inf.st.train.scripts.BenchmarkMain' + classpath = sourceSets.main.runtimeClasspath + args 'individual_incremental' +} + + +task generate(dependsOn: 'classes', type: JavaExec) { + group = 'Benchmark' + description = 'Generates the models for all benchmarks' + main = 'de.tudresden.inf.st.train.scripts.GenerateMain' + classpath = sourceSets.main.runtimeClasspath +} + +//task memory(dependsOn: 'classes', type: JavaExec) { +// group = 'Benchmark' +// main = 'MemoryScript' +// classpath = sourceSets.main.runtimeClasspath +//} +// +//task qpt(dependsOn: 'classes', type: JavaExec) { +// group = 'Benchmark' +// main = 'QueryPlanTester' +// classpath = sourceSets.main.runtimeClasspath +//} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkMain.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkMain.java new file mode 100644 index 000000000..8659f0313 --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkMain.java @@ -0,0 +1,140 @@ +package de.tudresden.inf.st.train.scripts; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.ObjectMapper; +import hu.bme.mit.trainbenchmark.benchmark.config.*; +import hu.bme.mit.trainbenchmark.benchmark.result.ResultHelper; +import hu.bme.mit.trainbenchmark.benchmark.runcomponents.BenchmarkRunner; +import hu.bme.mit.trainbenchmark.config.ExecutionConfig; +import hu.bme.mit.trainbenchmark.constants.RailwayOperation; + +import java.io.IOException; +import java.util.*; + +/** + * Main entry point to start benchmarks. + * Created by rschoene on 10/12/17. + */ +public class BenchmarkMain { + + @SuppressWarnings("unchecked") + private static void runBenchmarkSeries(BenchmarkConfigBaseBuilder configBaseBuilder, BenchmarkConfigBuilder configBuilder, + ExecutionConfig ec, ModelSetConfig modelSetConfig, BenchmarkSettingsBasics bbs) { + Map<String, String> jvmSettingsMap = Utils.createJvmSettingsMap(bbs); + String jvmDefaultSettings = jvmSettingsMap.getOrDefault("*", + "-Xms{Xms} -Xmx{Xmx} -server -Xverify:none"); + try { + for (int size = modelSetConfig.getMinSize(); size <= modelSetConfig.getMaxSize(); size *= 2) { + String modelFilename = "railway-" + modelSetConfig.getModelVariant() + "-" + size; + + System.out.println("------------------------------------------------------------"); + System.out.println("Model: " + modelFilename); + System.out.println("------------------------------------------------------------"); + + configBaseBuilder.setModelFilename(modelFilename); + BenchmarkConfigBase configBase = configBaseBuilder.createConfigBase(); + BenchmarkConfig config = (BenchmarkConfig) configBuilder.setConfigBase(configBase).createConfig(); + + if (!bbs.isDryRun()) { + ResultHelper.prepare(config, ec); + } + String jvmSetting = jvmSettingsMap.getOrDefault(config.getToolName(), jvmDefaultSettings); + + int exitValue = BenchmarkRunner.runPerformanceBenchmark(config, ec, jvmSetting); + if (exitValue != 0) { + System.out.println("Timeout or error occurred, skipping models for larger sizes. Error code: " + exitValue); + break; + } + } + } catch (InterruptedException | IOException e) { + System.out.println("Exception occurred during execution."); + e.printStackTrace(); + } + } + + public static void main(String args[]) throws Exception { + // set benchmark variant from program arguments + if (args.length == 0) { + System.err.println("Need to specify one argument identifying the benchmark variant"); + System.exit(1); + } + String benchmarkVariant = args[0]; + + // read basic settings (min, maxsize, etc.) + ObjectMapper mapper = Utils.getMapper(); + BenchmarkSettingsBasics bbs = Utils.readFromResource(mapper, "basic-settings.json", + BenchmarkSettingsBasics.class); + BenchmarkSettingsBasics loca_bbs = Utils.readFromResource(mapper, "local-basic-settings.json", + BenchmarkSettingsBasics.class); + Utils.updateBasicsWithLocal(bbs, loca_bbs); + + // import tools to use + List<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> builders = + BenchmarkSettingsAvailableTools.getBuilders(bbs.getTools()); + System.out.println(builders); + + // read workloads based on benchmark variant + String workloadsFilename = "workloads.json"; + BenchmarkSettingsVariantWorkloads bsvw = Utils.readFromResource(mapper, workloadsFilename, + BenchmarkSettingsVariantWorkloads.class); + + Optional<BenchmarkSettingsWorkloads> workloadsOpt = bsvw.getVariants().stream() + .filter(w -> w.getName().equals(benchmarkVariant)) + .findFirst(); + if (!workloadsOpt.isPresent()) { + System.err.println("Could not find workloads for given variant, check " + workloadsFilename + + " and argument " + benchmarkVariant); + System.exit(4); + } + BenchmarkSettingsWorkloads workloads = workloadsOpt.get(); + + // start benchmark + String now = ResultHelper.getNow(); + ExecutionConfig ec = new ExecutionConfig(4000, 8000, bbs.isDryRun()); + + System.out.println("Please remember to stop all other Java processes."); + System.out.println(); + System.out.println("If in doubt, check with this command:"); + System.out.println("$ ps auxw | grep jav[a]"); + System.out.println(); + System.out.println("If there are other Java processes, use:"); + System.out.println("$ killall -9 java"); + System.out.println(); + System.out.println("############################################################"); + System.out.println("Benchmark parameters:"); + System.out.println("- execution config: " + ec); + System.out.println(bbs); +// System.out.println("- range: minSize=" + bbs.getMinSize() +", maxSize=" + bbs.getMaxSize()); +// System.out.println("- timeout: " + bbs.getTimeout()); +// System.out.println("- runs: " + bbs.getRuns()); + System.out.println("############################################################"); + System.out.println(); + + workloads.getWorkloads().forEach(workloadConfiguration -> { + String workloadName = workloadConfiguration.getWorkloadName(); + String modelVariant = workloadConfiguration.getModelVariant(); + List<RailwayOperation> operations = workloadConfiguration.getOperations(); + TransformationChangeSetStrategy strategy = workloadConfiguration.getStrategy(); + int constant = workloadConfiguration.getConstant(); + int queryTransformationCount = workloadConfiguration.getQueryTransformationCount(); + + System.out.println("============================================================"); + System.out.println("Workload: " + workloadName); + System.out.println("============================================================"); + + ModelSetConfig modelSetConfig = new ModelSetConfig(modelVariant, bbs.getMinSize(), bbs.getMaxSize()); + BenchmarkConfigBaseBuilder bcbb = new BenchmarkConfigBaseBuilder() + .setBenchmarkId(now).setTimeout(bbs.getTimeout()).setRuns(bbs.getRuns()) + .setOperations(operations).setWorkload(workloadName) + .setQueryTransformationCount(queryTransformationCount).setTransformationConstant(constant) + .setTransformationChangeSetStrategy(strategy); + + builders.forEach( bcb -> runBenchmarkSeries(bcbb, bcb, ec, modelSetConfig, bbs)); + }); + +// if (binding.variables.get("reportUrl")) { +// BenchmarkReporter.reportReady(reportUrl) +// } + } + +} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableFormats.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableFormats.java new file mode 100644 index 000000000..4b93da1e6 --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableFormats.java @@ -0,0 +1,83 @@ +package de.tudresden.inf.st.train.scripts; + +import de.tudresden.inf.st.trainbenchmark.generator.json4ag.config.Json4AgGeneratorConfigBuilder; +import de.tudresden.inf.st.trainbenchmark.generator.json4agref.config.Json4AgRefGeneratorConfigBuilder; +import hu.bme.mit.trainbenchmark.generator.config.*; +import hu.bme.mit.trainbenchmark.generator.emf.config.EmfGeneratorConfigBuilder; +import hu.bme.mit.trainbenchmark.generator.graph.neo4j.config.Neo4jGraphGeneratorConfigBuilder; +import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphFormat; +import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphGeneratorConfigBuilder; +import hu.bme.mit.trainbenchmark.generator.rdf.config.RdfGeneratorConfigBuilder; +import hu.bme.mit.trainbenchmark.generator.sql.config.SqlGeneratorConfigBuilder; +import hu.bme.mit.trainbenchmark.neo4j.config.Neo4jGraphFormat; +import hu.bme.mit.trainbenchmark.rdf.RdfFormat; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * Format settings of a benchmark shared throughout all variants. + * Created by rschoene on 10/13/17. + */ +public class BenchmarkSettingsAvailableFormats { + /** + * Create a map, from projectName to a set of all builders in this project + * @return a mapping from projectName to all builders + */ + public static Map<String, List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>>> + getAvailableFormats() { + // { projectName -> [builder] } + GeneratorConfigBase dummyConfigBase = new GeneratorConfigBaseBuilder() + .setScenario(Scenario.BATCH).setSize(1) + .createGeneratorConfigBase(); + return getAllBuilders().stream().collect(Collectors.toMap( + builder -> builder.setConfigBase(dummyConfigBase).createConfig().getProjectName(), + s -> new ArrayList<>(Collections.singletonList(s)), + (oldValue, newValue) -> { oldValue.addAll(newValue); return oldValue; })); + } + + private static GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>> + getBuilderByName(String className) { + Optional<Object> result = Utils.maybeCreateNewInstance(className); + return (GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>) result.orElse(null); + } + + private static List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>> + getAllBuilders() { + return Arrays.asList( + new Neo4jGraphGeneratorConfigBuilder().setGraphFormat(Neo4jGraphFormat.CSV), + new Neo4jGraphGeneratorConfigBuilder().setGraphFormat(Neo4jGraphFormat.GRAPHML), + new EmfGeneratorConfigBuilder(), + new TinkerGraphGeneratorConfigBuilder().setGraphFormat(TinkerGraphFormat.GRAPHML), + new Json4AgGeneratorConfigBuilder(), + new Json4AgRefGeneratorConfigBuilder(), + new RdfGeneratorConfigBuilder().setFormat(RdfFormat.TURTLE).setInferred(true), + new RdfGeneratorConfigBuilder().setFormat(RdfFormat.TURTLE).setInferred(false), + new SqlGeneratorConfigBuilder() + ); + } + + /** + * Get all builders with the given names of formats. The format name in the config every builder creates + * must match one of the given project names. + * @param formatNames names of formats + * @return a set of matching generator config builders + */ + public static List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>> + getBuilders(String ... formatNames) { + Map<String, List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>>> + availableFormats = getAvailableFormats(); + return Arrays.stream(formatNames).map(name -> { + List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>> + builders = availableFormats.get(name); + if (builders.isEmpty()) { + throw new IllegalArgumentException("Could not find format " + name + ", available formats are: " + + availableFormats.keySet()); + } + return builders; + }).reduce((set1, set2) -> { + set1.addAll(set2); + return set1; + }).orElse(Collections.emptyList()); + } +} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableTools.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableTools.java new file mode 100644 index 000000000..1dc230051 --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableTools.java @@ -0,0 +1,94 @@ +package de.tudresden.inf.st.train.scripts; + +import de.tudresden.inf.st.train.kiama.config.KiamaBenchmarkConfigBuilder; +import de.tudresden.inf.st.train.racr.config.RacrBenchmarkConfigBuilder; +import hu.bme.mit.trainbenchmark.benchmark.config.*; +import hu.bme.mit.trainbenchmark.benchmark.tinkergraph.config.TinkerGraphBenchmarkConfigBuilder; +import hu.bme.mit.trainbenchmark.benchmark.viatra.config.ViatraBackend; +import hu.bme.mit.trainbenchmark.benchmark.viatra.config.ViatraBenchmarkConfigBuilder; + +import hu.bme.mit.trainbenchmark.benchmark.drools.config.DroolsBenchmarkConfigBuilder; +import hu.bme.mit.trainbenchmark.benchmark.eclipseocl.config.EclipseOclBenchmarkConfigBuilder; +import hu.bme.mit.trainbenchmark.benchmark.emfapi.config.EmfApiBenchmarkConfigBuilder; +import hu.bme.mit.trainbenchmark.benchmark.jena.config.JenaBenchmarkConfigBuilder; +import hu.bme.mit.trainbenchmark.benchmark.mysql.config.MySqlBenchmarkConfigBuilder; +import hu.bme.mit.trainbenchmark.benchmark.sqlite.config.SQLiteBenchmarkConfigBuilder; +import hu.bme.mit.trainbenchmark.benchmark.neo4j.config.Neo4jBenchmarkConfigBuilder; +import hu.bme.mit.trainbenchmark.neo4j.config.Neo4jGraphFormat; +import hu.bme.mit.trainbenchmark.benchmark.neo4j.config.Neo4jEngine; +import hu.bme.mit.trainbenchmark.benchmark.rdf4j.config.Rdf4jBenchmarkConfigBuilder; + +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Tool settings of a benchmark shared throughout all variants + * Created by rschoene on 10/12/17. + */ +public class BenchmarkSettingsAvailableTools { + public static Map<String, BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> + getAvailableTools() { + // { toolName -> builder } + BenchmarkConfigBase dummyConfigBase = new BenchmarkConfigBaseBuilder() + .setBenchmarkId("").setTimeout(0L).setRuns(0).setModelFilename("") + .setOperations(new ArrayList<>()).setWorkload("") + .setTransformationChangeSetStrategy(TransformationChangeSetStrategy.FIXED) + .setQueryTransformationCount(0).setTransformationConstant(0) + .createConfigBase(); + return getAllBuilders().stream().collect(Collectors.toMap( + builder -> builder.setConfigBase(dummyConfigBase).createConfig().getToolName(), + Function.identity())); + } + + private static BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>> + getBuilderByName(String className) { + Optional<Object> result = Utils.maybeCreateNewInstance(className); + return (BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>) result.orElse(null); + } + + private static List<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> + getAllBuilders() { + return Arrays.asList( + new DroolsBenchmarkConfigBuilder(), + new EclipseOclBenchmarkConfigBuilder(), + new EmfApiBenchmarkConfigBuilder(), + getBuilderByName("de.tudresden.inf.st.train.jastadd.config.JastaddBenchmarkConfigBuilder"), + getBuilderByName("de.tudresden.inf.st.train.jastadd.config.JastaddIncrementalBenchmarkConfigBuilder"), + getBuilderByName("de.tudresden.inf.st.train.jastadd.config.JastaddSymbolicBenchmarkConfigBuilder"), + getBuilderByName("de.tudresden.inf.st.train.jastadd.config.JastaddSymbolicIncrementalBenchmarkConfigBuilder"), + new JenaBenchmarkConfigBuilder().setInferencing(false), + new JenaBenchmarkConfigBuilder().setInferencing(true), + new KiamaBenchmarkConfigBuilder(), + new MySqlBenchmarkConfigBuilder(), + new Neo4jBenchmarkConfigBuilder().setEngine(Neo4jEngine.COREAPI).setGraphFormat(Neo4jGraphFormat.CSV ), + new Neo4jBenchmarkConfigBuilder().setEngine(Neo4jEngine.CYPHER ).setGraphFormat(Neo4jGraphFormat.GRAPHML), + new RacrBenchmarkConfigBuilder().setVerbose(false).useCpp().useInternalServer(), + new RacrBenchmarkConfigBuilder().setVerbose(false).usePython(), + new RacrBenchmarkConfigBuilder().setVerbose(false).useScheme(), + new Rdf4jBenchmarkConfigBuilder().setInferencing(false), + new SQLiteBenchmarkConfigBuilder(), + new TinkerGraphBenchmarkConfigBuilder(), + new ViatraBenchmarkConfigBuilder().setBackend(ViatraBackend.INCREMENTAL), + new ViatraBenchmarkConfigBuilder().setBackend(ViatraBackend.LOCAL_SEARCH) + ); + } + + /** + * Get all builders with the given names of tools. The tool name in the config every builder creates must match one + * of the given tool names. + * @param toolNames names of tools + * @return a list of matching benchmark config builders + */ + public static List<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> + getBuilders(String ... toolNames) { + Map<String, BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> availableTools = getAvailableTools(); + return Arrays.stream(toolNames).map(name -> { + BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>> builder = availableTools.get(name); + if (builder == null) { + throw new IllegalArgumentException("Could not find tool " + name + ", available tools are: " + availableTools.keySet()); + } + return builder; + }).collect(Collectors.toList()); + } +} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsBasics.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsBasics.java new file mode 100644 index 000000000..cf4270523 --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsBasics.java @@ -0,0 +1,134 @@ +package de.tudresden.inf.st.train.scripts; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonSetter; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Basic settings bean of a benchmark shared throughout all variants. + * Created by rschoene on 10/12/17. + */ +public class BenchmarkSettingsBasics { + + static class JVMSetting { + private List<String> applyTo; + private String args; + + public List<String> getApplyTo() { + return applyTo; + } + + public void setApplyTo(List<String> applyTo) { + this.applyTo = applyTo; + } + + public String getArgs() { + return args; + } + + public void setArgs(String args) { + this.args = args; + } + } + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + private Integer minSize = null; + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + private Integer maxSize = null; + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + private Long timeout = null; + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + private Integer runs = null; + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + private Boolean dryRun = null; + private String[] tools; + private String[] formats; + private List<JVMSetting> jvmSettings; + + /** Minimum model size */ + public Integer getMinSize() { + return minSize; + } + + public void setMinSize(int minSize) { + this.minSize = minSize; + } + + /** Maximum model size */ + public Integer getMaxSize() { + return maxSize; + } + + public void setMaxSize(int maxSize) { + this.maxSize = maxSize; + } + + /** Timeout in seconds for each single run. */ + public Long getTimeout() { + return timeout; + } + + public void setTimeout(long timeout) { + this.timeout = timeout; + } + + /** Number of runs per tool-workload combination. */ + public Integer getRuns() { + return runs; + } + + public void setRuns(int runs) { + this.runs = runs; + } + + /** Don't run any Java processes. */ + public Boolean isDryRun() { + return dryRun; + } + + @JsonSetter("dry-run") + public void setDryRun(boolean dryRun) { + this.dryRun = dryRun; + } + + /** List of tool names to use */ + public String[] getTools() { + return tools; + } + + public void setTools(String[] tools) { + this.tools = tools; + } + + /** List of format names to use */ + public String[] getFormats() { + return formats; + } + + public void setFormats(String[] formats) { + this.formats = formats; + } + + public List<JVMSetting> getJvmSettings() { + return jvmSettings; + } + + public void setJvmSettings(List<JVMSetting> jvmSettings) { + this.jvmSettings = jvmSettings; + } + + @Override + public String toString() { + return "BenchmarkSettingsBasics{" + + "minSize=" + minSize + + ", maxSize=" + maxSize + + ", timeout=" + timeout + + ", runs=" + runs + + ", dryRun=" + dryRun + + ", tools=" + Arrays.toString(tools) + + '}'; + } +} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsVariantWorkloads.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsVariantWorkloads.java new file mode 100644 index 000000000..ab89bcdf3 --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsVariantWorkloads.java @@ -0,0 +1,19 @@ +package de.tudresden.inf.st.train.scripts; + +import java.util.List; + +/** + * List of all workload settings by variant. + * Created by rschoene on 10/12/17. + */ +public class BenchmarkSettingsVariantWorkloads { + private List<BenchmarkSettingsWorkloads> variants; + + public List<BenchmarkSettingsWorkloads> getVariants() { + return variants; + } + + public void setVariants(List<BenchmarkSettingsWorkloads> variants) { + this.variants = variants; + } +} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsWorkload.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsWorkload.java new file mode 100644 index 000000000..95b00151a --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsWorkload.java @@ -0,0 +1,81 @@ +package de.tudresden.inf.st.train.scripts; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import hu.bme.mit.trainbenchmark.benchmark.config.TransformationChangeSetStrategy; +import hu.bme.mit.trainbenchmark.constants.RailwayOperation; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Benchmark setting workload bean. + * Created by rschoene on 10/12/17. + */ +public class BenchmarkSettingsWorkload { + private String workloadName; + private String modelVariant; + private List<RailwayOperation> operations; + private TransformationChangeSetStrategy strategy; +// private String strategy; + private int constant; + private int queryTransformationCount; + @JsonIgnore + private Map<String, RailwayOperation> railwayOps; + + public BenchmarkSettingsWorkload() { + railwayOps = Arrays.stream(RailwayOperation.values()).collect(Collectors.toMap( + RailwayOperation::toString, Function.identity() + )); + } + + public String getWorkloadName() { + return workloadName; + } + + public void setWorkloadName(String workloadName) { + this.workloadName = workloadName; + } + + public String getModelVariant() { + return modelVariant; + } + + public void setModelVariant(String modelVariant) { + this.modelVariant = modelVariant; + } + + public List<RailwayOperation> getOperations() { + return operations; + } + + public void setOperations(List<String> operations) { + this.operations = operations.stream().map(op -> railwayOps.get(op)).collect(Collectors.toList()); + } + + public TransformationChangeSetStrategy getStrategy() { + return strategy; + } + + public void setStrategy(TransformationChangeSetStrategy strategy) { + this.strategy = strategy; + } + + public int getConstant() { + return constant; + } + + public void setConstant(int constant) { + this.constant = constant; + } + + public int getQueryTransformationCount() { + return queryTransformationCount; + } + + public void setQueryTransformationCount(int queryTransformationCount) { + this.queryTransformationCount = queryTransformationCount; + } +} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsWorkloads.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsWorkloads.java new file mode 100644 index 000000000..d32c2290b --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsWorkloads.java @@ -0,0 +1,29 @@ +package de.tudresden.inf.st.train.scripts; + +import java.util.List; + +/** + * Workload settings bean. + * Created by rschoene on 10/12/17. + */ +public class BenchmarkSettingsWorkloads +{ + private String name; + private List<BenchmarkSettingsWorkload> workloads; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public List<BenchmarkSettingsWorkload> getWorkloads() { + return workloads; + } + + public void setWorkloads(List<BenchmarkSettingsWorkload> workloads) { + this.workloads = workloads; + } +} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkVariant.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkVariant.java new file mode 100644 index 000000000..d6f04f118 --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkVariant.java @@ -0,0 +1,12 @@ +package de.tudresden.inf.st.train.scripts; + +/** + * Possible variants to run a benchmark in. + * Created by rschoene on 10/12/17. + */ +public enum BenchmarkVariant { + COMBINED, + INDIVIDUAL_INJECT, + INDIVIDUAL_REPAIR, + INDIVIDUAL_INCREMENTAL +} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/GenerateMain.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/GenerateMain.java new file mode 100644 index 000000000..052019c49 --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/GenerateMain.java @@ -0,0 +1,108 @@ +package de.tudresden.inf.st.train.scripts; + +import com.fasterxml.jackson.databind.ObjectMapper; +import hu.bme.mit.trainbenchmark.config.ExecutionConfig; +import hu.bme.mit.trainbenchmark.generator.config.*; +import hu.bme.mit.trainbenchmark.generator.runner.GeneratorRunner; + +import java.util.List; + +/** + * Main entry point to generate models. + * Created by rschoene on 10/13/17. + */ +public class GenerateMain { + public static void main(String[] args) throws Exception { + // read basic settings (min, maxsize, etc.) + ObjectMapper mapper = Utils.getMapper(); + BenchmarkSettingsBasics bbs = Utils.readFromResource(mapper, "basic-settings.json", + BenchmarkSettingsBasics.class); + BenchmarkSettingsBasics loca_bbs = Utils.readFromResource(mapper, "local-basic-settings.json", + BenchmarkSettingsBasics.class); + Utils.updateBasicsWithLocal(bbs, loca_bbs); + + // import formats to use + List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>> builders = + BenchmarkSettingsAvailableFormats.getBuilders(bbs.getFormats()); + System.out.println(builders); + ExecutionConfig ec = new ExecutionConfig(4000, 8000, bbs.isDryRun()); + + // start generating (exclude Scenario.MINIMAL) + for (Scenario scenario : new Scenario[]{Scenario.BATCH, Scenario.INJECT, Scenario.REPAIR}) { + builders.forEach( generatorConfigBuilder -> { + try { + for (int size = bbs.getMinSize(); size <= bbs.getMaxSize(); size *= 2) { + System.out.println("Scenario: " + scenario + ", size: " + size); + + GeneratorConfigBase configBase = new GeneratorConfigBaseBuilder() + .setSize(size).setScenario(scenario) + .createGeneratorConfigBase(); + GeneratorConfig config = generatorConfigBuilder.setConfigBase(configBase).createConfig(); + + int exitValue = GeneratorRunner.run(config, ec); + if (exitValue != 0) { + System.out.println("Timeout or error occurred, skipping models for larger sizes. Error code: " + + exitValue); + break; + } + } + } catch (Exception e) { + System.out.println("Exception occurred during execution."); + } + }); + } + } + +} + +/* +import de.tudresden.inf.st.trainbenchmark.generator.json4ag.config.Json4AgGeneratorConfigBuilder +import de.tudresden.inf.st.trainbenchmark.generator.json4agref.config.Json4AgRefGeneratorConfigBuilder +import hu.bme.mit.trainbenchmark.config.ExecutionConfig +import hu.bme.mit.trainbenchmark.generator.config.GeneratorConfigBase +import hu.bme.mit.trainbenchmark.generator.config.Scenario +import hu.bme.mit.trainbenchmark.generator.emf.config.EmfGeneratorConfigBuilder +import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphFormat +import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphGeneratorConfigBuilder +import hu.bme.mit.trainbenchmark.generator.runner.GeneratorRunner + +def ec = new ExecutionConfig(4000, 6000) +def minSize = 1 +/* def maxSize = 2048 * +def maxSize = 256 + + def scenarios = [ + Scenario.BATCH, + Scenario.INJECT, + Scenario.REPAIR, + ] + + def formats = [ + new EmfGeneratorConfigBuilder(), + new TinkerGraphGeneratorConfigBuilder().setGraphFormat(TinkerGraphFormat.GRAPHML), + new Json4AgGeneratorConfigBuilder(), + new Json4AgRefGeneratorConfigBuilder() + ] + + for (scenario in scenarios) { + formats.each { generatorConfigBuilder -> + try { + for (def size = minSize; size <= maxSize; size *= 2) { + println("Scenario: ${scenario}, size: ${size}") + + def configBase = new GeneratorConfigBase(scenario, size) + def config = generatorConfigBuilder.setConfigBase(configBase).createConfig() + + def exitValue = GeneratorRunner.run(config, ec) + if (exitValue != 0) { + println "Timeout or error occured, skipping models for larger sizes. Error code: ${exitValue}" + break + } + } + } catch (all) { + println "Exception occured during execution." + } + } + } + + */ diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/Utils.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/Utils.java new file mode 100644 index 000000000..e155896b5 --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/Utils.java @@ -0,0 +1,99 @@ +package de.tudresden.inf.st.train.scripts; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.URL; +import java.util.*; +import java.util.stream.Collectors; + +/** + * Utility methods for the main entry points. + * Created by rschoene on 10/13/17. + */ +public class Utils { + static Map<String, String> createJvmSettingsMap(BenchmarkSettingsBasics bbs) { + Map<String, String> result = new HashMap<>(); + for (BenchmarkSettingsBasics.JVMSetting setting : bbs.getJvmSettings()) { + for (String toolName : setting.getApplyTo()) { + result.put(toolName, setting.getArgs()); + } + } + return result; + } + + private static File readFromResource(String filename) throws IOException { + URL basicSettingsURL = BenchmarkMain.class.getClassLoader().getResource(filename); + if (basicSettingsURL == null) { + System.err.println(); + throw new IOException("Could not access " + filename + ". Exiting."); + } + return new File(basicSettingsURL.getFile()); + } + + static <T> T readFromResource(ObjectMapper mapper, String filename, Class<T> clazz) throws IOException { + File basicSettingsFile = readFromResource(filename); + T result = null; + try { + result = mapper.readValue(basicSettingsFile, clazz); + } catch (Exception e) { + System.err.println("Could not load '" + filename + "'. Exiting."); + e.printStackTrace(); + System.exit(2); + } + return result; + } + + static void updateBasicsWithLocal(BenchmarkSettingsBasics base, BenchmarkSettingsBasics local) + throws IllegalStateException { + List<Method> setter = Arrays.stream(BenchmarkSettingsBasics.class.getMethods()) + .filter(m -> m.getName().startsWith("set")) + .collect(Collectors.toList()); + for (Method method : BenchmarkSettingsBasics.class.getMethods()) { + final String name = method.getName().startsWith("get") ? + method.getName().substring(3) : + ( method.getName().startsWith("is") ? method.getName().substring(2) : null); + if (name == null) { + continue; + } + try { + Object result = method.invoke(local); + if (result != null) { + Optional<Method> matchingSetter = setter.stream() + .filter(m -> m.getName().endsWith(name)) + .findFirst(); + if (matchingSetter.isPresent()) { + matchingSetter.get().invoke(base, result); + } + } + // now do sanity check if set in updated base + result = method.invoke(base); + if (result == null) { + throw new IllegalStateException("Basic setting for " + name + " is not given."); + } + } catch (IllegalAccessException | InvocationTargetException e) { + e.printStackTrace(); + } + } + } + + static ObjectMapper getMapper() { + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true); + return mapper; + } + + static Optional<Object> maybeCreateNewInstance(final String className) { + try { + return Optional.of(Class.forName(className).newInstance()); + } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { + System.err.println("Could not find builder class '" + className + "'"); + e.printStackTrace(); + return Optional.empty(); + } + } +} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/.gitignore b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/.gitignore new file mode 100644 index 000000000..4ee2202e5 --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/.gitignore @@ -0,0 +1,2 @@ +# basic settings can still be commited using --force +local-basic-settings.json diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/basic-settings.json b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/basic-settings.json new file mode 100644 index 000000000..6d6b000ca --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/basic-settings.json @@ -0,0 +1,49 @@ +{ + "minSize": 1, + "maxSize": 8, + "timeout": 900, + "runs": 1, + "dry-run": true, + "tools": [ + "Drools", + "Eclipse OCL", + "EMF API", + "Jena (Inferencing)", + "Jena (No Inferencing)", + "MySQL", + "Neo4j (Core API-CSV)", + "Neo4j (Cypher-GraphML)", + "Racr (Scheme)", + "Racr (Python)", + "Racr (CPP)", + "RDF4J (No Inferencing)", + "SQLite", + "Jastadd (Symbolic References)", + "Jastadd (Incremental + Symbolic References)", + "Jastadd (Java References)", + "Jastadd (Incremental + Java References)", + "Kiama", + "TinkerGraph", + "VIATRA (Incremental)", + "VIATRA (Local Search)" + ], + "formats": [ + "emf", + "json4ag", + "json4ag-ref", + "graph-tinkerpop", + "rdf", + "sql" + ], + "jvmSettings": [ + { + "applyTo": ["*"], + "args": "-Xms%(Xms) -Xmx%(Xmx) -server -Xverify:none" +// "args": "-Xms%(Xms) -Xmx%(Xmx) -server -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:StartFlightRecording=dumponexit=true -Xverify:none" + }, + { + "applyTo": ["Racr (CPP)", "Racr (Python)", "Racr (Scheme)"], + "args": "-Xms50M -Xmx50M -server -Xverify:none" + } + ] +} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/workloads.json b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/workloads.json new file mode 100644 index 000000000..513e70203 --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/workloads.json @@ -0,0 +1,258 @@ +{ + "variants": [ + { + "name": "individual_repair", + "workloads": [ + { + "workloadName": "ConnectedSegments", + "operations": [ + "ConnectedSegmentsRepair" + ], + "constant": 5, + "queryTransformationCount": 8, + "modelVariant": "repair", + "strategy": "PROPORTIONAL" + }, + { + "workloadName": "PosLength", + "operations": [ + "PosLengthRepair" + ], + "constant": 5, + "queryTransformationCount": 8, + "modelVariant": "repair", + "strategy": "PROPORTIONAL" + }, + { + "workloadName": "RouteSensor", + "operations": [ + "RouteSensorRepair" + ], + "constant": 5, + "queryTransformationCount": 8, + "modelVariant": "repair", + "strategy": "PROPORTIONAL" + }, + { + "workloadName": "SemaphoreNeighbor", + "operations": [ + "SemaphoreNeighborRepair" + ], + "constant": 5, + "queryTransformationCount": 8, + "modelVariant": "repair", + "strategy": "PROPORTIONAL" + }, + { + "workloadName": "SwitchMonitored", + "operations": [ + "SwitchMonitoredRepair" + ], + "constant": 5, + "queryTransformationCount": 8, + "modelVariant": "repair", + "strategy": "PROPORTIONAL" + }, + { + "workloadName": "SwitchSet", + "operations": [ + "SwitchSetRepair" + ], + "constant": 5, + "queryTransformationCount": 8, + "modelVariant": "repair", + "strategy": "PROPORTIONAL" + } + ] + }, + { + "name": "combined", + "workloads": [ + { + "workloadName": "Repair", + "operations": [ + "ConnectedSegmentsRepair", + "PosLengthRepair", + "RouteSensorRepair", + "SemaphoreNeighborRepair", + "SwitchMonitoredRepair", + "SwitchSetRepair" + ], + "constant": 5, + "queryTransformationCount": 8, + "modelVariant": "repair", + "strategy": "PROPORTIONAL" + }, + { + "workloadName": "Inject", + "operations": [ + "ConnectedSegments", + "PosLength", + "RouteSensor", + "SemaphoreNeighbor", + "SwitchMonitored", + "SwitchSet", + "ConnectedSegmentsInject", + "PosLengthInject", + "RouteSensorInject", + "SemaphoreNeighborInject", + "SwitchMonitoredInject", + "SwitchSetInject" + ], + "constant": 10, + "queryTransformationCount": 12, + "modelVariant": "inject", + "strategy": "FIXED" + } + ] + }, + { + "name": "individual_inject", + "workloads": [ + { + "workloadName": "ConnectedSegments", + "operations": [ + "ConnectedSegments", + "ConnectedSegmentsInject" + ], + "constant": 10, + "queryTransformationCount": 12, + "modelVariant": "inject", + "strategy": "FIXED" + }, + { + "workloadName": "PosLength", + "operations": [ + "PosLength", + "PosLengthInject" + ], + "constant": 10, + "queryTransformationCount": 12, + "modelVariant": "inject", + "strategy": "FIXED" + }, + { + "workloadName": "RouteSensor", + "operations": [ + "RouteSensor", + "RouteSensorInject" + ], + "constant": 10, + "queryTransformationCount": 12, + "modelVariant": "inject", + "strategy": "FIXED" + }, + { + "workloadName": "SemaphoreNeighbor", + "operations": [ + "SemaphoreNeighbor", + "SemaphoreNeighborInject" + ], + "constant": 10, + "queryTransformationCount": 12, + "modelVariant": "inject", + "strategy": "FIXED" + }, + { + "workloadName": "SwitchMonitored", + "operations": [ + "SwitchMonitored", + "SwitchMonitoredInject" + ], + "constant": 10, + "queryTransformationCount": 12, + "modelVariant": "inject", + "strategy": "FIXED" + }, + { + "workloadName": "SwitchSet", + "operations": [ + "SwitchSet", + "SwitchSetInject" + ], + "constant": 10, + "queryTransformationCount": 12, + "modelVariant": "inject", + "strategy": "FIXED" + } + ] + }, + { + "name": "individual_incremental", + "workloads": [ + { + "workloadName": "ConnectedSegments", + "operations": [ + "ConnectedSegments", + "ConnectedSegmentsInject", + "ConnectedSegmentsRepair" + ], + "constant": 10, + "queryTransformationCount": 50, + "modelVariant": "inject", + "strategy": "FIXED" + }, + { + "workloadName": "PosLength", + "operations": [ + "PosLength", + "PosLengthInject", + "PosLengthRepair" + ], + "constant": 10, + "queryTransformationCount": 50, + "modelVariant": "inject", + "strategy": "FIXED" + }, + { + "workloadName": "RouteSensor", + "operations": [ + "RouteSensor", + "RouteSensorInject", + "RouteSensorRepair" + ], + "constant": 10, + "queryTransformationCount": 50, + "modelVariant": "inject", + "strategy": "FIXED" + }, + { + "workloadName": "SemaphoreNeighbor", + "operations": [ + "SemaphoreNeighbor", + "SemaphoreNeighborInject", + "SemaphoreNeighborRepair" + ], + "constant": 10, + "queryTransformationCount": 50, + "modelVariant": "inject", + "strategy": "FIXED" + }, + { + "workloadName": "SwitchMonitored", + "operations": [ + "SwitchMonitored", + "SwitchMonitoredInject", + "SwitchMonitoredRepair" + ], + "constant": 10, + "queryTransformationCount": 50, + "modelVariant": "inject", + "strategy": "FIXED" + }, + { + "workloadName": "SwitchSet", + "operations": [ + "SwitchSet", + "SwitchSetInject", + "SwitchSetRepair" + ], + "constant": 10, + "queryTransformationCount": 50, + "modelVariant": "inject", + "strategy": "FIXED" + } + ] + } + ] +} -- GitLab