diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/build.gradle b/trainbenchmark/trainbenchmark-alternate-scripts/build.gradle new file mode 100644 index 0000000000000000000000000000000000000000..fc879099b2d9c4ca1b83fd0cc48db6657cce75ad --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/build.gradle @@ -0,0 +1,83 @@ +dependencies { + compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.2.3' + compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.3.0' + + compile project(':trainbenchmark-generator') + compile project(':trainbenchmark-generator-emf') + compile project(':trainbenchmark-generator-graph-tinkerpop') + compile project(':trainbenchmark-generator-json4ag') + compile project(':trainbenchmark-generator-dot') + compile project(':trainbenchmark-tool') + compile project(':trainbenchmark-tool-jastadd-base') + compile project(':trainbenchmark-tool-jastadd-namelookup') + compile project(':trainbenchmark-tool-jastadd-namelookup-base') + compile project(':trainbenchmark-tool-jastadd-namelookup-incremental') + compile project(':trainbenchmark-tool-jastadd-optimized') + compile project(':trainbenchmark-tool-jastadd-optimized-base') + compile project(':trainbenchmark-tool-jastadd-optimized-incremental') +// compile project(':trainbenchmark-tool-jastadd-specialized') +// compile project(':trainbenchmark-tool-jastadd-specialized-base') +// compile project(':trainbenchmark-tool-jastadd-specialized-incremental') +// compile project(':trainbenchmark-tool-jastadd-relast') +// compile project(':trainbenchmark-tool-jastadd-relast-incremental') + compile project(':trainbenchmark-tool-tinkergraph') + compile project(':trainbenchmark-tool-viatra') +} + +task cleanResults(type: Delete) { + doLast { + def resultDir = "../results" + file(resultDir).eachDir{ dir -> + delete "${dir}" + } + } +} + +task combinedBenchmark(dependsOn: 'classes', type: JavaExec) { + group = 'Benchmark' + description = 'Runs the \'classic\' TrainBenchmark script' + main = 'de.tudresden.inf.st.train.scripts.BenchmarkMain' + classpath = sourceSets.main.runtimeClasspath + args 'combined' +} + +task individualRepairBenchmark(dependsOn: 'classes', type: JavaExec) { + group = 'Benchmark' + main = 'de.tudresden.inf.st.train.scripts.BenchmarkMain' + classpath = sourceSets.main.runtimeClasspath + args 'individual_repair' +} + +task individualInjectBenchmark(dependsOn: 'classes', type: JavaExec) { + group = 'Benchmark' + main = 'de.tudresden.inf.st.train.scripts.BenchmarkMain' + classpath = sourceSets.main.runtimeClasspath + args 'individual_inject' +} + +task individualIncrementalBenchmark(dependsOn: 'classes', type: JavaExec) { + group = 'Benchmark' + main = 'de.tudresden.inf.st.train.scripts.BenchmarkMain' + classpath = sourceSets.main.runtimeClasspath + args 'individual_incremental' +} + + +task generate(dependsOn: 'classes', type: JavaExec) { + group = 'Benchmark' + description = 'Generates the models for all benchmarks' + main = 'de.tudresden.inf.st.train.scripts.GenerateMain' + classpath = sourceSets.main.runtimeClasspath +} + +//task memory(dependsOn: 'classes', type: JavaExec) { +// group = 'Benchmark' +// main = 'MemoryScript' +// classpath = sourceSets.main.runtimeClasspath +//} +// +//task qpt(dependsOn: 'classes', type: JavaExec) { +// group = 'Benchmark' +// main = 'QueryPlanTester' +// classpath = sourceSets.main.runtimeClasspath +//} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkMain.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkMain.java new file mode 100644 index 0000000000000000000000000000000000000000..c5dca00cfa62bd4fec9cd314ef0672b37949e1a8 --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkMain.java @@ -0,0 +1,144 @@ +package de.tudresden.inf.st.train.scripts; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.ObjectMapper; +import hu.bme.mit.trainbenchmark.benchmark.config.*; +import hu.bme.mit.trainbenchmark.benchmark.result.ResultHelper; +import hu.bme.mit.trainbenchmark.benchmark.runcomponents.BenchmarkRunner; +import hu.bme.mit.trainbenchmark.config.ExecutionConfig; +import hu.bme.mit.trainbenchmark.constants.RailwayOperation; + +import java.io.IOException; +import java.util.*; + +/** + * Main entry point to start benchmarks. + * Created by rschoene on 10/12/17. + */ +public class BenchmarkMain { + + @SuppressWarnings("unchecked") + private static void runBenchmarkSeries(BenchmarkConfigBaseBuilder configBaseBuilder, BenchmarkConfigBuilder configBuilder, + ExecutionConfig ec, ModelSetConfig modelSetConfig, BenchmarkSettingsBasics bbs) { + Map<String, String> jvmSettingsMap = Utils.createJvmSettingsMap(bbs); + String jvmDefaultSettings = jvmSettingsMap.getOrDefault("*", + "-Xms{Xms} -Xmx{Xmx} -server -Xverify:none"); + try { + for (int size = modelSetConfig.getMinSize(); size <= modelSetConfig.getMaxSize(); size *= 2) { + String modelFilename = "railway-" + modelSetConfig.getModelVariant() + "-" + size; + + System.out.println("------------------------------------------------------------"); + System.out.println("Model: " + modelFilename); + System.out.println("------------------------------------------------------------"); + + configBaseBuilder.setModelFilename(modelFilename); + BenchmarkConfigBase configBase = configBaseBuilder.createConfigBase(); + BenchmarkConfig config = (BenchmarkConfig) configBuilder.setConfigBase(configBase).createConfig(); + + if (!bbs.isDryRun()) { + ResultHelper.prepare(config, ec); + } + String jvmSetting = jvmSettingsMap.getOrDefault(config.getToolName(), jvmDefaultSettings); + + int exitValue = BenchmarkRunner.runPerformanceBenchmark(config, ec, jvmSetting); + if (exitValue != 0) { + System.out.println("Timeout or error occurred, skipping models for larger sizes. Error code: " + exitValue); + break; + } + } + } catch (InterruptedException | IOException e) { + System.out.println("Exception occurred during execution."); + e.printStackTrace(); + } + } + + public static void main(String args[]) throws Exception { + // set benchmark variant from program arguments + if (args.length == 0) { + System.err.println("Need to specify one argument identifying the benchmark variant"); + System.exit(1); + } + String benchmarkVariant = args[0]; + + // read basic settings (min, maxsize, etc.) + ObjectMapper mapper = Utils.getMapper(); + BenchmarkSettingsBasics bbs = Utils.readFromResource(mapper, "basic-settings.json", + BenchmarkSettingsBasics.class); + try { + BenchmarkSettingsBasics loca_bbs = Utils.readFromResource(mapper, "local-basic-settings.json", + BenchmarkSettingsBasics.class); + Utils.updateBasicsWithLocal(bbs, loca_bbs); + } catch (IOException e) { + System.out.println("No file 'local-basic-settings.json' found. Using default!"); + } + + // import tools to use + List<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> builders = + BenchmarkSettingsAvailableTools.getBuilders(bbs.getTools()); + System.out.println(builders); + + // read workloads based on benchmark variant + String workloadsFilename = "workloads.json"; + BenchmarkSettingsVariantWorkloads bsvw = Utils.readFromResource(mapper, workloadsFilename, + BenchmarkSettingsVariantWorkloads.class); + + Optional<BenchmarkSettingsWorkloads> workloadsOpt = bsvw.getVariants().stream() + .filter(w -> w.getName().equals(benchmarkVariant)) + .findFirst(); + if (!workloadsOpt.isPresent()) { + System.err.println("Could not find workloads for given variant, check " + workloadsFilename + + " and argument " + benchmarkVariant); + System.exit(4); + } + BenchmarkSettingsWorkloads workloads = workloadsOpt.get(); + + // start benchmark + String now = ResultHelper.getNow(); + ExecutionConfig ec = new ExecutionConfig(4000, 8000, bbs.isDryRun()); + + System.out.println("Please remember to stop all other Java processes."); + System.out.println(); + System.out.println("If in doubt, check with this command:"); + System.out.println("$ ps auxw | grep jav[a]"); + System.out.println(); + System.out.println("If there are other Java processes, use:"); + System.out.println("$ killall -9 java"); + System.out.println(); + System.out.println("############################################################"); + System.out.println("Benchmark parameters:"); + System.out.println("- execution config: " + ec); + System.out.println(bbs); +// System.out.println("- range: minSize=" + bbs.getMinSize() +", maxSize=" + bbs.getMaxSize()); +// System.out.println("- timeout: " + bbs.getTimeout()); +// System.out.println("- runs: " + bbs.getRuns()); + System.out.println("############################################################"); + System.out.println(); + + workloads.getWorkloads().forEach(workloadConfiguration -> { + String workloadName = workloadConfiguration.getWorkloadName(); + String modelVariant = workloadConfiguration.getModelVariant(); + List<RailwayOperation> operations = workloadConfiguration.getOperations(); + TransformationChangeSetStrategy strategy = workloadConfiguration.getStrategy(); + int constant = workloadConfiguration.getConstant(); + int queryTransformationCount = workloadConfiguration.getQueryTransformationCount(); + + System.out.println("============================================================"); + System.out.println("Workload: " + workloadName); + System.out.println("============================================================"); + + ModelSetConfig modelSetConfig = new ModelSetConfig(modelVariant, bbs.getMinSize(), bbs.getMaxSize()); + BenchmarkConfigBaseBuilder bcbb = new BenchmarkConfigBaseBuilder() + .setBenchmarkId(now).setTimeout(bbs.getTimeout()).setRuns(bbs.getRuns()) + .setOperations(operations).setWorkload(workloadName) + .setQueryTransformationCount(queryTransformationCount).setTransformationConstant(constant) + .setTransformationChangeSetStrategy(strategy); + + builders.forEach( bcb -> runBenchmarkSeries(bcbb, bcb, ec, modelSetConfig, bbs)); + }); + +// if (binding.variables.get("reportUrl")) { +// BenchmarkReporter.reportReady(reportUrl) +// } + } + +} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableFormats.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableFormats.java new file mode 100644 index 0000000000000000000000000000000000000000..439a0801d10840f94907902fb355864c04a7d921 --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableFormats.java @@ -0,0 +1,71 @@ +package de.tudresden.inf.st.train.scripts; + +import de.tudresden.inf.st.trainbenchmark.generator.json4ag.config.Json4AgGeneratorConfigBuilder; +import hu.bme.mit.trainbenchmark.generator.config.*; +import hu.bme.mit.trainbenchmark.generator.emf.config.EmfGeneratorConfigBuilder; +import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphFormat; +import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphGeneratorConfigBuilder; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * Format settings of a benchmark shared throughout all variants. + * Created by rschoene on 10/13/17. + */ +public class BenchmarkSettingsAvailableFormats { + /** + * Create a map, from projectName to a set of all builders in this project + * @return a mapping from projectName to all builders + */ + public static Map<String, List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>>> + getAvailableFormats() { + // { projectName -> [builder] } + GeneratorConfigBase dummyConfigBase = new GeneratorConfigBaseBuilder() + .setScenario(Scenario.BATCH).setSize(1) + .createGeneratorConfigBase(); + return getAllBuilders().stream().collect(Collectors.toMap( + builder -> builder.setConfigBase(dummyConfigBase).createConfig().getProjectName(), + s -> new ArrayList<>(Collections.singletonList(s)), + (oldValue, newValue) -> { oldValue.addAll(newValue); return oldValue; })); + } + + private static GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>> + getBuilderByName(String className) { + Optional<Object> result = Utils.maybeCreateNewInstance(className); + return (GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>) result.orElse(null); + } + + private static List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>> + getAllBuilders() { + return Arrays.asList( + new EmfGeneratorConfigBuilder(), + new TinkerGraphGeneratorConfigBuilder().setGraphFormat(TinkerGraphFormat.GRAPHML), + new Json4AgGeneratorConfigBuilder() + ); + } + + /** + * Get all builders with the given names of formats. The format name in the config every builder creates + * must match one of the given project names. + * @param formatNames names of formats + * @return a set of matching generator config builders + */ + public static List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>> + getBuilders(String ... formatNames) { + Map<String, List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>>> + availableFormats = getAvailableFormats(); + return Arrays.stream(formatNames).map(name -> { + List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>> + builders = availableFormats.get(name); + if (builders.isEmpty()) { + throw new IllegalArgumentException("Could not find format " + name + ", available formats are: " + + availableFormats.keySet()); + } + return builders; + }).reduce((set1, set2) -> { + set1.addAll(set2); + return set1; + }).orElse(Collections.emptyList()); + } +} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableTools.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableTools.java new file mode 100644 index 0000000000000000000000000000000000000000..49dc323b78a5cbb8f5a12c2e9326b5b3c642f2ab --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsAvailableTools.java @@ -0,0 +1,71 @@ +package de.tudresden.inf.st.train.scripts; + +import de.tudresden.inf.st.train.jastadd.config.JastaddNameLookupBenchmarkConfigBuilder; +import de.tudresden.inf.st.train.jastadd.config.JastaddNameLookupIncrementalBenchmarkConfigBuilder; +import de.tudresden.inf.st.train.jastadd.config.JastaddOptimizedBenchmarkConfigBuilder; +import de.tudresden.inf.st.train.jastadd.config.JastaddOptimizedIncrementalBenchmarkConfigBuilder; +import hu.bme.mit.trainbenchmark.benchmark.config.*; +import hu.bme.mit.trainbenchmark.benchmark.tinkergraph.config.TinkerGraphBenchmarkConfigBuilder; +import hu.bme.mit.trainbenchmark.benchmark.viatra.config.ViatraBackend; +import hu.bme.mit.trainbenchmark.benchmark.viatra.config.ViatraBenchmarkConfigBuilder; + +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Tool settings of a benchmark shared throughout all variants + * Created by rschoene on 10/12/17. + */ +public class BenchmarkSettingsAvailableTools { + public static Map<String, BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> + getAvailableTools() { + // { toolName -> builder } + BenchmarkConfigBase dummyConfigBase = new BenchmarkConfigBaseBuilder() + .setBenchmarkId("").setTimeout(0L).setRuns(0).setModelFilename("") + .setOperations(new ArrayList<>()).setWorkload("") + .setTransformationChangeSetStrategy(TransformationChangeSetStrategy.FIXED) + .setQueryTransformationCount(0).setTransformationConstant(0) + .createConfigBase(); + return getAllBuilders().stream().collect(Collectors.toMap( + builder -> builder.setConfigBase(dummyConfigBase).createConfig().getToolName(), + Function.identity())); + } + + private static BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>> + getBuilderByName(String className) { + Optional<Object> result = Utils.maybeCreateNewInstance(className); + return (BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>) result.orElse(null); + } + + private static List<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> + getAllBuilders() { + return Arrays.asList( + new JastaddNameLookupBenchmarkConfigBuilder(), + new JastaddNameLookupIncrementalBenchmarkConfigBuilder(), + new JastaddOptimizedBenchmarkConfigBuilder(), + new JastaddOptimizedIncrementalBenchmarkConfigBuilder(), + new TinkerGraphBenchmarkConfigBuilder(), + new ViatraBenchmarkConfigBuilder().setBackend(ViatraBackend.INCREMENTAL), + new ViatraBenchmarkConfigBuilder().setBackend(ViatraBackend.LOCAL_SEARCH) + ); + } + + /** + * Get all builders with the given names of tools. The tool name in the config every builder creates must match one + * of the given tool names. + * @param toolNames names of tools + * @return a list of matching benchmark config builders + */ + public static List<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> + getBuilders(String ... toolNames) { + Map<String, BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> availableTools = getAvailableTools(); + return Arrays.stream(toolNames).map(name -> { + BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>> builder = availableTools.get(name); + if (builder == null) { + throw new IllegalArgumentException("Could not find tool " + name + ", available tools are: " + availableTools.keySet()); + } + return builder; + }).collect(Collectors.toList()); + } +} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsBasics.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsBasics.java new file mode 100644 index 0000000000000000000000000000000000000000..cf427052309cfb2da890c8eacd7077fddf2d1645 --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsBasics.java @@ -0,0 +1,134 @@ +package de.tudresden.inf.st.train.scripts; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonSetter; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Basic settings bean of a benchmark shared throughout all variants. + * Created by rschoene on 10/12/17. + */ +public class BenchmarkSettingsBasics { + + static class JVMSetting { + private List<String> applyTo; + private String args; + + public List<String> getApplyTo() { + return applyTo; + } + + public void setApplyTo(List<String> applyTo) { + this.applyTo = applyTo; + } + + public String getArgs() { + return args; + } + + public void setArgs(String args) { + this.args = args; + } + } + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + private Integer minSize = null; + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + private Integer maxSize = null; + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + private Long timeout = null; + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + private Integer runs = null; + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + private Boolean dryRun = null; + private String[] tools; + private String[] formats; + private List<JVMSetting> jvmSettings; + + /** Minimum model size */ + public Integer getMinSize() { + return minSize; + } + + public void setMinSize(int minSize) { + this.minSize = minSize; + } + + /** Maximum model size */ + public Integer getMaxSize() { + return maxSize; + } + + public void setMaxSize(int maxSize) { + this.maxSize = maxSize; + } + + /** Timeout in seconds for each single run. */ + public Long getTimeout() { + return timeout; + } + + public void setTimeout(long timeout) { + this.timeout = timeout; + } + + /** Number of runs per tool-workload combination. */ + public Integer getRuns() { + return runs; + } + + public void setRuns(int runs) { + this.runs = runs; + } + + /** Don't run any Java processes. */ + public Boolean isDryRun() { + return dryRun; + } + + @JsonSetter("dry-run") + public void setDryRun(boolean dryRun) { + this.dryRun = dryRun; + } + + /** List of tool names to use */ + public String[] getTools() { + return tools; + } + + public void setTools(String[] tools) { + this.tools = tools; + } + + /** List of format names to use */ + public String[] getFormats() { + return formats; + } + + public void setFormats(String[] formats) { + this.formats = formats; + } + + public List<JVMSetting> getJvmSettings() { + return jvmSettings; + } + + public void setJvmSettings(List<JVMSetting> jvmSettings) { + this.jvmSettings = jvmSettings; + } + + @Override + public String toString() { + return "BenchmarkSettingsBasics{" + + "minSize=" + minSize + + ", maxSize=" + maxSize + + ", timeout=" + timeout + + ", runs=" + runs + + ", dryRun=" + dryRun + + ", tools=" + Arrays.toString(tools) + + '}'; + } +} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsVariantWorkloads.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsVariantWorkloads.java new file mode 100644 index 0000000000000000000000000000000000000000..ab89bcdf3a68b80e9ecac6525b0ed7fd43a867c0 --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsVariantWorkloads.java @@ -0,0 +1,19 @@ +package de.tudresden.inf.st.train.scripts; + +import java.util.List; + +/** + * List of all workload settings by variant. + * Created by rschoene on 10/12/17. + */ +public class BenchmarkSettingsVariantWorkloads { + private List<BenchmarkSettingsWorkloads> variants; + + public List<BenchmarkSettingsWorkloads> getVariants() { + return variants; + } + + public void setVariants(List<BenchmarkSettingsWorkloads> variants) { + this.variants = variants; + } +} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsWorkload.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsWorkload.java new file mode 100644 index 0000000000000000000000000000000000000000..95b00151ae26fab885202a58e77f63d1086f3a04 --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsWorkload.java @@ -0,0 +1,81 @@ +package de.tudresden.inf.st.train.scripts; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import hu.bme.mit.trainbenchmark.benchmark.config.TransformationChangeSetStrategy; +import hu.bme.mit.trainbenchmark.constants.RailwayOperation; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Benchmark setting workload bean. + * Created by rschoene on 10/12/17. + */ +public class BenchmarkSettingsWorkload { + private String workloadName; + private String modelVariant; + private List<RailwayOperation> operations; + private TransformationChangeSetStrategy strategy; +// private String strategy; + private int constant; + private int queryTransformationCount; + @JsonIgnore + private Map<String, RailwayOperation> railwayOps; + + public BenchmarkSettingsWorkload() { + railwayOps = Arrays.stream(RailwayOperation.values()).collect(Collectors.toMap( + RailwayOperation::toString, Function.identity() + )); + } + + public String getWorkloadName() { + return workloadName; + } + + public void setWorkloadName(String workloadName) { + this.workloadName = workloadName; + } + + public String getModelVariant() { + return modelVariant; + } + + public void setModelVariant(String modelVariant) { + this.modelVariant = modelVariant; + } + + public List<RailwayOperation> getOperations() { + return operations; + } + + public void setOperations(List<String> operations) { + this.operations = operations.stream().map(op -> railwayOps.get(op)).collect(Collectors.toList()); + } + + public TransformationChangeSetStrategy getStrategy() { + return strategy; + } + + public void setStrategy(TransformationChangeSetStrategy strategy) { + this.strategy = strategy; + } + + public int getConstant() { + return constant; + } + + public void setConstant(int constant) { + this.constant = constant; + } + + public int getQueryTransformationCount() { + return queryTransformationCount; + } + + public void setQueryTransformationCount(int queryTransformationCount) { + this.queryTransformationCount = queryTransformationCount; + } +} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsWorkloads.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsWorkloads.java new file mode 100644 index 0000000000000000000000000000000000000000..d32c2290b4be746cfe5963619a1f07dc968d9592 --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkSettingsWorkloads.java @@ -0,0 +1,29 @@ +package de.tudresden.inf.st.train.scripts; + +import java.util.List; + +/** + * Workload settings bean. + * Created by rschoene on 10/12/17. + */ +public class BenchmarkSettingsWorkloads +{ + private String name; + private List<BenchmarkSettingsWorkload> workloads; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public List<BenchmarkSettingsWorkload> getWorkloads() { + return workloads; + } + + public void setWorkloads(List<BenchmarkSettingsWorkload> workloads) { + this.workloads = workloads; + } +} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkVariant.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkVariant.java new file mode 100644 index 0000000000000000000000000000000000000000..d6f04f1183bf0cd409f770347a87f6d769f15276 --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/BenchmarkVariant.java @@ -0,0 +1,12 @@ +package de.tudresden.inf.st.train.scripts; + +/** + * Possible variants to run a benchmark in. + * Created by rschoene on 10/12/17. + */ +public enum BenchmarkVariant { + COMBINED, + INDIVIDUAL_INJECT, + INDIVIDUAL_REPAIR, + INDIVIDUAL_INCREMENTAL +} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/GenerateMain.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/GenerateMain.java new file mode 100644 index 0000000000000000000000000000000000000000..367c5f5f0e3b73014640f242af5df01c4cbd6496 --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/GenerateMain.java @@ -0,0 +1,113 @@ +package de.tudresden.inf.st.train.scripts; + +import com.fasterxml.jackson.databind.ObjectMapper; +import hu.bme.mit.trainbenchmark.config.ExecutionConfig; +import hu.bme.mit.trainbenchmark.generator.config.*; +import hu.bme.mit.trainbenchmark.generator.runner.GeneratorRunner; + +import java.io.IOException; +import java.util.List; + +/** + * Main entry point to generate models. + * Created by rschoene on 10/13/17. + */ +public class GenerateMain { + public static void main(String[] args) throws Exception { + // read basic settings (min, maxsize, etc.) + ObjectMapper mapper = Utils.getMapper(); + BenchmarkSettingsBasics bbs = Utils.readFromResource(mapper, "basic-settings.json", + BenchmarkSettingsBasics.class); + try { + BenchmarkSettingsBasics loca_bbs = Utils.readFromResource(mapper, "local-basic-settings.json", + BenchmarkSettingsBasics.class); + Utils.updateBasicsWithLocal(bbs, loca_bbs); + } catch (IOException e) { + System.out.println("No file 'local-basic-settings.json' found. Using default!"); + } + + // import formats to use + List<GeneratorConfigBuilder<? extends GeneratorConfig, ? extends GeneratorConfigBuilder<?, ?>>> builders = + BenchmarkSettingsAvailableFormats.getBuilders(bbs.getFormats()); + System.out.println(builders); + ExecutionConfig ec = new ExecutionConfig(4000, 8000, bbs.isDryRun()); + + // start generating (exclude Scenario.MINIMAL) + for (Scenario scenario : new Scenario[]{Scenario.BATCH, Scenario.INJECT, Scenario.REPAIR}) { + builders.forEach( generatorConfigBuilder -> { + try { + for (int size = bbs.getMinSize(); size <= bbs.getMaxSize(); size *= 2) { + System.out.println("Scenario: " + scenario + ", size: " + size); + + GeneratorConfigBase configBase = new GeneratorConfigBaseBuilder() + .setSize(size).setScenario(scenario) + .createGeneratorConfigBase(); + GeneratorConfig config = generatorConfigBuilder.setConfigBase(configBase).createConfig(); + + int exitValue = GeneratorRunner.run(config, ec); + if (exitValue != 0) { + System.out.println("Timeout or error occurred, skipping models for larger sizes. Error code: " + + exitValue); + break; + } + } + } catch (Exception e) { + System.out.println("Exception occurred during execution."); + } + }); + } + } + +} + +/* +import de.tudresden.inf.st.trainbenchmark.generator.json4ag.config.Json4AgGeneratorConfigBuilder +import de.tudresden.inf.st.trainbenchmark.generator.json4agref.config.Json4AgRefGeneratorConfigBuilder +import hu.bme.mit.trainbenchmark.config.ExecutionConfig +import hu.bme.mit.trainbenchmark.generator.config.GeneratorConfigBase +import hu.bme.mit.trainbenchmark.generator.config.Scenario +import hu.bme.mit.trainbenchmark.generator.emf.config.EmfGeneratorConfigBuilder +import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphFormat +import hu.bme.mit.trainbenchmark.generator.graph.tinkerpop.config.TinkerGraphGeneratorConfigBuilder +import hu.bme.mit.trainbenchmark.generator.runner.GeneratorRunner + +def ec = new ExecutionConfig(4000, 6000) +def minSize = 1 +/* def maxSize = 2048 * +def maxSize = 256 + + def scenarios = [ + Scenario.BATCH, + Scenario.INJECT, + Scenario.REPAIR, + ] + + def formats = [ + new EmfGeneratorConfigBuilder(), + new TinkerGraphGeneratorConfigBuilder().setGraphFormat(TinkerGraphFormat.GRAPHML), + new Json4AgGeneratorConfigBuilder(), + new Json4AgRefGeneratorConfigBuilder() + ] + + for (scenario in scenarios) { + formats.each { generatorConfigBuilder -> + try { + for (def size = minSize; size <= maxSize; size *= 2) { + println("Scenario: ${scenario}, size: ${size}") + + def configBase = new GeneratorConfigBase(scenario, size) + def config = generatorConfigBuilder.setConfigBase(configBase).createConfig() + + def exitValue = GeneratorRunner.run(config, ec) + if (exitValue != 0) { + println "Timeout or error occured, skipping models for larger sizes. Error code: ${exitValue}" + break + } + } + } catch (all) { + println "Exception occured during execution." + } + } + } + + */ diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/Utils.java b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/Utils.java new file mode 100644 index 0000000000000000000000000000000000000000..e155896b5f1f21927c3328f37ed596288e8cd9fb --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/java/de/tudresden/inf/st/train/scripts/Utils.java @@ -0,0 +1,99 @@ +package de.tudresden.inf.st.train.scripts; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.net.URL; +import java.util.*; +import java.util.stream.Collectors; + +/** + * Utility methods for the main entry points. + * Created by rschoene on 10/13/17. + */ +public class Utils { + static Map<String, String> createJvmSettingsMap(BenchmarkSettingsBasics bbs) { + Map<String, String> result = new HashMap<>(); + for (BenchmarkSettingsBasics.JVMSetting setting : bbs.getJvmSettings()) { + for (String toolName : setting.getApplyTo()) { + result.put(toolName, setting.getArgs()); + } + } + return result; + } + + private static File readFromResource(String filename) throws IOException { + URL basicSettingsURL = BenchmarkMain.class.getClassLoader().getResource(filename); + if (basicSettingsURL == null) { + System.err.println(); + throw new IOException("Could not access " + filename + ". Exiting."); + } + return new File(basicSettingsURL.getFile()); + } + + static <T> T readFromResource(ObjectMapper mapper, String filename, Class<T> clazz) throws IOException { + File basicSettingsFile = readFromResource(filename); + T result = null; + try { + result = mapper.readValue(basicSettingsFile, clazz); + } catch (Exception e) { + System.err.println("Could not load '" + filename + "'. Exiting."); + e.printStackTrace(); + System.exit(2); + } + return result; + } + + static void updateBasicsWithLocal(BenchmarkSettingsBasics base, BenchmarkSettingsBasics local) + throws IllegalStateException { + List<Method> setter = Arrays.stream(BenchmarkSettingsBasics.class.getMethods()) + .filter(m -> m.getName().startsWith("set")) + .collect(Collectors.toList()); + for (Method method : BenchmarkSettingsBasics.class.getMethods()) { + final String name = method.getName().startsWith("get") ? + method.getName().substring(3) : + ( method.getName().startsWith("is") ? method.getName().substring(2) : null); + if (name == null) { + continue; + } + try { + Object result = method.invoke(local); + if (result != null) { + Optional<Method> matchingSetter = setter.stream() + .filter(m -> m.getName().endsWith(name)) + .findFirst(); + if (matchingSetter.isPresent()) { + matchingSetter.get().invoke(base, result); + } + } + // now do sanity check if set in updated base + result = method.invoke(base); + if (result == null) { + throw new IllegalStateException("Basic setting for " + name + " is not given."); + } + } catch (IllegalAccessException | InvocationTargetException e) { + e.printStackTrace(); + } + } + } + + static ObjectMapper getMapper() { + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(JsonParser.Feature.ALLOW_COMMENTS, true); + return mapper; + } + + static Optional<Object> maybeCreateNewInstance(final String className) { + try { + return Optional.of(Class.forName(className).newInstance()); + } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { + System.err.println("Could not find builder class '" + className + "'"); + e.printStackTrace(); + return Optional.empty(); + } + } +} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/.gitignore b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..4ee2202e58823ac10973a1d236e6404a8c11b4ef --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/.gitignore @@ -0,0 +1,2 @@ +# basic settings can still be commited using --force +local-basic-settings.json diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/basic-settings.json b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/basic-settings.json new file mode 100644 index 0000000000000000000000000000000000000000..501ab8fc36d9f4507c97454d9dbc36e37b7637b7 --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/basic-settings.json @@ -0,0 +1,32 @@ +{ + "minSize": 1, + "maxSize": 8, + "timeout": 900, + "runs": 1, + "dry-run": true, + "tools": [ + "Jastadd (Name Lookup)", + "Jastadd (Incremental Name Lookup)", + "Jastadd (Optimized)", + "Jastadd (Optimized Incremental)", + "TinkerGraph", + "VIATRA (Incremental)", + "VIATRA (Local Search)" + ], + "formats": [ + "emf", + "json4ag", + "graph-tinkerpop" + ], + "jvmSettings": [ + { + "applyTo": ["*"], + "args": "-Xms%(Xms) -Xmx%(Xmx) -server -Xverify:none" +// "args": "-Xms%(Xms) -Xmx%(Xmx) -server -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:StartFlightRecording=dumponexit=true -Xverify:none" + }, + { + "applyTo": ["Racr (CPP)", "Racr (Python)", "Racr (Scheme)"], + "args": "-Xms50M -Xmx50M -server -Xverify:none" + } + ] +} diff --git a/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/workloads.json b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/workloads.json new file mode 100644 index 0000000000000000000000000000000000000000..513e70203ba17237c104e4dd9f72b1510dabadcd --- /dev/null +++ b/trainbenchmark/trainbenchmark-alternate-scripts/src/main/resources/workloads.json @@ -0,0 +1,258 @@ +{ + "variants": [ + { + "name": "individual_repair", + "workloads": [ + { + "workloadName": "ConnectedSegments", + "operations": [ + "ConnectedSegmentsRepair" + ], + "constant": 5, + "queryTransformationCount": 8, + "modelVariant": "repair", + "strategy": "PROPORTIONAL" + }, + { + "workloadName": "PosLength", + "operations": [ + "PosLengthRepair" + ], + "constant": 5, + "queryTransformationCount": 8, + "modelVariant": "repair", + "strategy": "PROPORTIONAL" + }, + { + "workloadName": "RouteSensor", + "operations": [ + "RouteSensorRepair" + ], + "constant": 5, + "queryTransformationCount": 8, + "modelVariant": "repair", + "strategy": "PROPORTIONAL" + }, + { + "workloadName": "SemaphoreNeighbor", + "operations": [ + "SemaphoreNeighborRepair" + ], + "constant": 5, + "queryTransformationCount": 8, + "modelVariant": "repair", + "strategy": "PROPORTIONAL" + }, + { + "workloadName": "SwitchMonitored", + "operations": [ + "SwitchMonitoredRepair" + ], + "constant": 5, + "queryTransformationCount": 8, + "modelVariant": "repair", + "strategy": "PROPORTIONAL" + }, + { + "workloadName": "SwitchSet", + "operations": [ + "SwitchSetRepair" + ], + "constant": 5, + "queryTransformationCount": 8, + "modelVariant": "repair", + "strategy": "PROPORTIONAL" + } + ] + }, + { + "name": "combined", + "workloads": [ + { + "workloadName": "Repair", + "operations": [ + "ConnectedSegmentsRepair", + "PosLengthRepair", + "RouteSensorRepair", + "SemaphoreNeighborRepair", + "SwitchMonitoredRepair", + "SwitchSetRepair" + ], + "constant": 5, + "queryTransformationCount": 8, + "modelVariant": "repair", + "strategy": "PROPORTIONAL" + }, + { + "workloadName": "Inject", + "operations": [ + "ConnectedSegments", + "PosLength", + "RouteSensor", + "SemaphoreNeighbor", + "SwitchMonitored", + "SwitchSet", + "ConnectedSegmentsInject", + "PosLengthInject", + "RouteSensorInject", + "SemaphoreNeighborInject", + "SwitchMonitoredInject", + "SwitchSetInject" + ], + "constant": 10, + "queryTransformationCount": 12, + "modelVariant": "inject", + "strategy": "FIXED" + } + ] + }, + { + "name": "individual_inject", + "workloads": [ + { + "workloadName": "ConnectedSegments", + "operations": [ + "ConnectedSegments", + "ConnectedSegmentsInject" + ], + "constant": 10, + "queryTransformationCount": 12, + "modelVariant": "inject", + "strategy": "FIXED" + }, + { + "workloadName": "PosLength", + "operations": [ + "PosLength", + "PosLengthInject" + ], + "constant": 10, + "queryTransformationCount": 12, + "modelVariant": "inject", + "strategy": "FIXED" + }, + { + "workloadName": "RouteSensor", + "operations": [ + "RouteSensor", + "RouteSensorInject" + ], + "constant": 10, + "queryTransformationCount": 12, + "modelVariant": "inject", + "strategy": "FIXED" + }, + { + "workloadName": "SemaphoreNeighbor", + "operations": [ + "SemaphoreNeighbor", + "SemaphoreNeighborInject" + ], + "constant": 10, + "queryTransformationCount": 12, + "modelVariant": "inject", + "strategy": "FIXED" + }, + { + "workloadName": "SwitchMonitored", + "operations": [ + "SwitchMonitored", + "SwitchMonitoredInject" + ], + "constant": 10, + "queryTransformationCount": 12, + "modelVariant": "inject", + "strategy": "FIXED" + }, + { + "workloadName": "SwitchSet", + "operations": [ + "SwitchSet", + "SwitchSetInject" + ], + "constant": 10, + "queryTransformationCount": 12, + "modelVariant": "inject", + "strategy": "FIXED" + } + ] + }, + { + "name": "individual_incremental", + "workloads": [ + { + "workloadName": "ConnectedSegments", + "operations": [ + "ConnectedSegments", + "ConnectedSegmentsInject", + "ConnectedSegmentsRepair" + ], + "constant": 10, + "queryTransformationCount": 50, + "modelVariant": "inject", + "strategy": "FIXED" + }, + { + "workloadName": "PosLength", + "operations": [ + "PosLength", + "PosLengthInject", + "PosLengthRepair" + ], + "constant": 10, + "queryTransformationCount": 50, + "modelVariant": "inject", + "strategy": "FIXED" + }, + { + "workloadName": "RouteSensor", + "operations": [ + "RouteSensor", + "RouteSensorInject", + "RouteSensorRepair" + ], + "constant": 10, + "queryTransformationCount": 50, + "modelVariant": "inject", + "strategy": "FIXED" + }, + { + "workloadName": "SemaphoreNeighbor", + "operations": [ + "SemaphoreNeighbor", + "SemaphoreNeighborInject", + "SemaphoreNeighborRepair" + ], + "constant": 10, + "queryTransformationCount": 50, + "modelVariant": "inject", + "strategy": "FIXED" + }, + { + "workloadName": "SwitchMonitored", + "operations": [ + "SwitchMonitored", + "SwitchMonitoredInject", + "SwitchMonitoredRepair" + ], + "constant": 10, + "queryTransformationCount": 50, + "modelVariant": "inject", + "strategy": "FIXED" + }, + { + "workloadName": "SwitchSet", + "operations": [ + "SwitchSet", + "SwitchSetInject", + "SwitchSetRepair" + ], + "constant": 10, + "queryTransformationCount": 50, + "modelVariant": "inject", + "strategy": "FIXED" + } + ] + } + ] +} diff --git a/trainbenchmark/trainbenchmark-config/src/main/java/hu/bme/mit/trainbenchmark/config/ExecutionConfig.java b/trainbenchmark/trainbenchmark-config/src/main/java/hu/bme/mit/trainbenchmark/config/ExecutionConfig.java index fa0ba9c21b13c1a7e1f5341ecd2a9e23f87f242c..a828d68d792bd8bdea8e49016ac93cea45740d05 100644 --- a/trainbenchmark/trainbenchmark-config/src/main/java/hu/bme/mit/trainbenchmark/config/ExecutionConfig.java +++ b/trainbenchmark/trainbenchmark-config/src/main/java/hu/bme/mit/trainbenchmark/config/ExecutionConfig.java @@ -4,14 +4,20 @@ public class ExecutionConfig { protected Integer initialMemory; protected Integer maxMemory; + protected Boolean dryRun; - public ExecutionConfig(final Integer initialMemory, final Integer maxMemory) { + public ExecutionConfig(final Integer initialMemory, final Integer maxMemory, final boolean dryRun) { this.initialMemory = initialMemory; this.maxMemory = maxMemory; + this.dryRun = dryRun; + } + + public ExecutionConfig(final Integer initialMemory, final Integer maxMemory) { + this(initialMemory, maxMemory, true); } /** - * + * * @return The initial memory for the benchmark JVM in MBs. */ public Integer getInitialMemory() { @@ -41,7 +47,15 @@ public class ExecutionConfig { public String getXmx() { return maxMemory + "M"; } - + + /** + * + * @return whether this benchmark run must not start any Java processes + */ + public Boolean getDryRun() { + return dryRun; + } + public static ExecutionConfig defaultExecutionConfig() { return new ExecutionConfig(1000, 1000); } diff --git a/trainbenchmark/trainbenchmark-reporting/.gitignore b/trainbenchmark/trainbenchmark-reporting/.gitignore index fb37a9f266860443b1f59ea9af4a8ce2eb089bce..5bfbad9180280146565a6e3c0127cf72266b33b7 100644 --- a/trainbenchmark/trainbenchmark-reporting/.gitignore +++ b/trainbenchmark/trainbenchmark-reporting/.gitignore @@ -1,2 +1,2 @@ Rplots.pdf - +local-merge_results.json diff --git a/trainbenchmark/trainbenchmark-reporting/build.gradle b/trainbenchmark/trainbenchmark-reporting/build.gradle index 5e8f0e2e1aa1f7b6240ab43977b0e13d2eccf0ac..7fc02588564bd4cedce73c2ecfc52d06fb27f0e3 100644 --- a/trainbenchmark/trainbenchmark-reporting/build.gradle +++ b/trainbenchmark/trainbenchmark-reporting/build.gradle @@ -1,7 +1,26 @@ +task doMerge(type: Exec) { + group = 'Benchmark' + description = 'Merges the results' + commandLine './do-merge.sh' +} + task plot(type: Exec) { - commandLine 'Rscript', 'report.R' + group = 'Benchmark' + description = 'Plots the \'classic\' TrainBenchmark result' + commandLine 'Rscript', 'report.R' + dependsOn doMerge } task plotIndividual(type: Exec) { - commandLine 'Rscript', 'individual.R' -} \ No newline at end of file + group = 'Benchmark' + description = 'Plots the individual TrainBenchmark results' + commandLine 'Rscript', 'individual.R' + dependsOn doMerge +} + +task plotToolwise(type: Exec) { + group = 'Benchmark' + description = 'Plots the individual TrainBenchmark results per tool' + commandLine './toolwise.sh' + dependsOn doMerge +} diff --git a/trainbenchmark/trainbenchmark-reporting/do-merge.sh b/trainbenchmark/trainbenchmark-reporting/do-merge.sh new file mode 100755 index 0000000000000000000000000000000000000000..cda5cf12b0fecef96942b21b2faf09054e7e097d --- /dev/null +++ b/trainbenchmark/trainbenchmark-reporting/do-merge.sh @@ -0,0 +1 @@ +python merge_results.py --result-dir ../results/ --create-run-dirs --create-toolwise-dirs $@ diff --git a/trainbenchmark/trainbenchmark-reporting/merge_results.json b/trainbenchmark/trainbenchmark-reporting/merge_results.json new file mode 100644 index 0000000000000000000000000000000000000000..d8de85321024170135a651f07de7db5d5c36a8d8 --- /dev/null +++ b/trainbenchmark/trainbenchmark-reporting/merge_results.json @@ -0,0 +1,27 @@ +{ + "tools": [ + "tinkergraph", + "drools", + "mysql", + "kiama", + "jastadd-java-references", + "jastadd-java-references-incremental", + "jastadd-symbolic-references-incremental", + "jastadd-symbolic-references", + "sqlite", + "viatra", + "racr-cpp", + "racr-python", + "racr-scheme", + "neo4j", + "sesame", + "emfapi", + "rdf4j", + "epsilon", + "eclipseocl" + ], + "ignored": [ + ], + "toolwise": [ + ] +} diff --git a/trainbenchmark/trainbenchmark-reporting/merge_results.py b/trainbenchmark/trainbenchmark-reporting/merge_results.py new file mode 100755 index 0000000000000000000000000000000000000000..a1fdb731eee2fab010b4f47498529971383bb359 --- /dev/null +++ b/trainbenchmark/trainbenchmark-reporting/merge_results.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python +import argparse +import csv +import glob +import json +import logging +import os +import os.path +import re +import sys + + +FORMAT = '%(asctime)s %(levelname)-8s %(threadName)-10s (%(filename)s:%(lineno)d): %(message)s' +BENCHMARK_PATTERN = re.compile('.*-(BatchModel|Repair|Inject)Test.*') +logger = logging.getLogger('merge_result') +SIZE_PATTERN = re.compile('.*-railway-[^\\-]*-([^\\-]*)-.csv') +NAME_PATTERN = re.compile('(times|matches)-([^\\-]*)-.*.csv') +RUN_PATTERN = re.compile('run-(....)-(..)-(..)-(..)-(..)-(..)') +RUN_REPLACMENT = r'\1_\2_\3 \4:\5:\6' + + +def include_file_config(args): + def override_if_defined(key, convert=lambda x: x): + keyArgs = key.replace('-', '_') + value = content.get(key) or content.get(keyArgs) + if value: + setattr(args, keyArgs, convert(value)) + + # load config file + with open(args.file_config) as fdr: + content = json.load(fdr) + # update with local version, if existing + directory, basename = os.path.split(os.path.abspath(args.file_config)) + local_config_file = os.path.join(directory, 'local-' + basename) + if os.path.exists(local_config_file): + with open(local_config_file) as fdr: + content.update(json.load(fdr)) + else: + logger.debug('No local config file found.') + if not content.get('tools'): + logger.error('Key "tools" not found in config file "' + args.file_config + '". Exiting.') + sys.exit(1) + args.tools = content['tools'] + override_if_defined('max-size', int) + override_if_defined('dry-run', bool) + override_if_defined('result-dir') + override_if_defined('create-run-dirs', bool) + override_if_defined('create-toolwise-dirs', bool) + override_if_defined('no-clean', bool) + override_if_defined('verbose', bool) + + +def create_link(fileToLink, linkName, dry_run): + if dry_run: + return + if os.path.lexists(linkName): + os.unlink(linkName) + (logger.info if args.dry_run else logger.debug)('Linking %s to %s', fileToLink, linkName) + os.symlink(fileToLink, linkName) + + +def ensure_directory(dir_name, dry_run): + if dry_run: + return + if not os.path.exists(dir_name): + logger.info('Creating %s', dir_name) + os.mkdir(dir_name) + + +def exceeds(filename, max_size): + match = SIZE_PATTERN.match(filename) + return int(match.group(1)) > max_size if match else False + + +def remove_if_there(the_list, element_to_remove): + if element_to_remove in the_list: + the_list.remove(element_to_remove) + + +def new_run_name(old_run_name): + return RUN_PATTERN.sub(RUN_REPLACMENT, old_run_name) + + +def copy_replace(fileTocopy, all_runs_dir, tool_name, run, dry_run): + """ + Take fileTocopy, copy it to all_runs_dir, while replacing tool_name with run + in both its name and its content + """ + run_name = new_run_name(run) + targetFile = os.path.join( + all_runs_dir, os.path.basename(fileTocopy).replace(tool_name, run_name)) + first = True + with open(fileTocopy) as fdr_source, open(targetFile, 'w') as fdr_target: + for line in fdr_source: + if first: + first = False + else: + line = line.replace(tool_name, run_name) + fdr_target.write(line) + + +def main(args): + """ + Main process. + + Used directory structure/variables: + results/ -> result_dir + tools/ - + tool1/ -> tool_dir + tool1-run1/ -> run_dir + times.csv - + matches.csv - + run-list.csv - + all-runs/ -> all_runs_dir + times-run1.csv@ - + run1/ -> global_run_dir + merged/ -> merged_dir + individual/ -> merged_dir_individual + times.csv@ - + combined/ -> merged_dir_benchmark + times.csv@ - + """ + log_action = logger.info if args.dry_run else logger.debug + + # Gathering paths, creating top-level directories + result_dir = os.path.abspath(args.result_dir) + merged_dir = os.path.join(result_dir, 'merged') + merged_dir_benchmark = os.path.join(merged_dir, 'benchmark') + merged_dir_individual = os.path.join(merged_dir, 'individual') + for dir_name in (merged_dir, merged_dir_benchmark, merged_dir_individual): + ensure_directory(dir_name, args.dry_run) + + # Gathering tools + tools = [] + reader = csv.reader(args.tools) + next(reader) + for row in reader: + if not row: + continue + tools.append(row[0]) + logger.debug('result_dir: %s, tools: %s', result_dir, tools) + + # Clean symlinks if requested or max_size is set + if (args.clean or args.max_size) and not args.dry_run: + for dir_to_clean in [merged_dir, merged_dir_benchmark, merged_dir_individual]: + for link in os.listdir(dir_to_clean): + linkName = os.path.join(dir_to_clean, link) + if os.path.islink(linkName): + os.unlink(linkName) + if (args.clean or args.create_toolwise_dirs) and not args.dry_run: + for linkName in glob.iglob(os.path.join(result_dir, 'tools', '*', 'all-runs', '*.csv')): + os.remove(linkName) + + # Merge results + for tool in tools: + if tool.startswith('#'): + logger.debug('Ignoring tool "%s"', tool[1:]) + continue + already_merged = [] + tool_dir = os.path.join(result_dir, 'tools', tool) + if not os.path.exists(tool_dir): + logger.warn('Tool not found: %s', tool) + continue + all_runs_dir = os.path.join(tool_dir, 'all-runs') + ensure_directory(all_runs_dir, args.dry_run) + runs = sorted(os.listdir(tool_dir), reverse=True) + remove_if_there(runs, 'all-runs') + remove_if_there(runs, 'run-list.csv') + + if args.create_toolwise_dirs: + # write out run-list.csv + with open(os.path.join(tool_dir, 'run-list.csv'), 'w') as fdr: + fdr.write('Runs\n') + for run in runs: + fdr.write(new_run_name(run) + '\n') + + for run in runs: + run_dir = os.path.join(tool_dir, run) + global_run_dir = os.path.join(result_dir, run) + if not os.path.isdir(run_dir): + continue + ensure_directory(global_run_dir, args.dry_run) + for csvFile in os.listdir(run_dir): + # link file in run directory + fileToLink = os.path.join(tool_dir, run, csvFile) + linkName = os.path.join(global_run_dir, csvFile) + create_link(fileToLink, linkName, args.dry_run) + # skip if max-size is set and size is exceeded + if args.max_size and exceeds(fileToLink, args.max_size): + continue + if args.create_toolwise_dirs: + # link in all-runs (rename file accordingly) + match = NAME_PATTERN.match(csvFile) + if match: + if not BENCHMARK_PATTERN.match(csvFile): + tool_name = match.group(2) + copy_replace(fileToLink, all_runs_dir, tool_name, run, args.dry_run) + else: + logging.warn('file did not match pattern: %s', csvFile) + # link file in merged directory + if csvFile not in already_merged: + linkName = os.path.join(merged_dir_benchmark if BENCHMARK_PATTERN.match(csvFile) + else merged_dir_individual, csvFile) + create_link(fileToLink, linkName, args.dry_run) + already_merged.append(csvFile) + else: + log_action('Skipping %s', csvFile) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Merge results of all benchmark runs.', + epilog="""The config file must contain the key "tools" specifying the tools to process. + It further can contain any long version of arguments to this program as a default value. + Any command line parameter will override such a default value. + Additionally, a local version of the file will be read, overriding the default values. + Its filename is "local-" prepended to the name of the config file.""") + parser.add_argument( + "-r", "--result-dir", help="Path to result directory to search in.", type=str) + parser.add_argument( + "-c", "--create-run-dirs", help="Whether to recreate runs directories.", + action="store_true") + parser.add_argument( + "-t", "--create-toolwise-dirs", help="Whether to recreate toolwise regression directories.", + action="store_true") + parser.add_argument( + "-d", "--dry-run", help="Only print action, don't execute them.", + action="store_true") + parser.add_argument( + "-n", "--no-clean", help="Don't remove previously existing symlinks in merged dir.", + dest='clean', action="store_false") + parser.add_argument( + "-v", "--verbose", help="Print debug messages.", action="store_true") + parser.add_argument( + "-m", "--max-size", type=int, + help="Maximum benchmark size to include. Implies cleaning existing symlinks.") + parser.add_argument( + "-f", "--file-config", default='merge_results.json', help="Config file to use.") + args = parser.parse_args() + include_file_config(args) + logging.basicConfig(format=FORMAT, level=logging.DEBUG if args.verbose else logging.INFO) + main(args) diff --git a/trainbenchmark/trainbenchmark-reporting/toolwise.R b/trainbenchmark/trainbenchmark-reporting/toolwise.R new file mode 100644 index 0000000000000000000000000000000000000000..3d0db01f39fa4eb0a3d6950c03ef4d8dd6444afc --- /dev/null +++ b/trainbenchmark/trainbenchmark-reporting/toolwise.R @@ -0,0 +1,154 @@ +library(data.table) +library(reshape2) +library(plyr) +library(ggplot2) +library(ggrepel) +library(arules) +library(ggforce) + +source('util.R') + +args = commandArgs(trailingOnly=TRUE) +if (length(args)==0) { + stop("At least one argument must be supplied (tool-name).\n", call.=FALSE) +} +toolName = args[1] + +# prepare output directory +output_dir = paste("../diagrams/merged", toolName, sep="/") +if (!(dir.exists(output_dir))) { + dir.create(output_dir) +} + +# constants +workloads = c( + "PosLength", "SwitchMonitored", + "RouteSensor", "SwitchSet", + "ConnectedSegments", "SemaphoreNeighbor" +) +phases = c("Read", "Check", "Read.and.Check", "Transformation", "Recheck", "Transformation.and.Recheck") +phasesPrettified = c("Read", "Check", "Read and Check", "Transformation", "Recheck", "Transformation and Recheck") + +sizes = list() # 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 +sizes[["Repair"]] = c("8k", "15k", "33k", "66k", "135k", "271k", "566k", "1.1M", "2.2M", "4.6M", "9.3M", "18M", "37M") + +runList = read.csv(paste("../results/tools/", toolName, "/run-list.csv", sep=""), colClasses=c(rep("character",1))) + +# load the data +tsvs = list.files(paste("../results/tools/", toolName, "/all-runs/", sep=""), pattern = "times-.*\\.csv", full.names = T, recursive = T) + +l = lapply(tsvs, read.csv) +times = rbindlist(l) + +# preprocess the data +times$Tool = factor(times$Tool, levels = runList$Runs) +keep_descriptions_first_char(times) + +times$Model = gsub("\\D+", "", times$Model) +times$Model = as.numeric(times$Model) +times$Time = times$Time / 10^6 +# make the phases a factor with a fixed set of values to help dcasting +# (e.g. Batch measurements do not have Transformation and Recheck attributes, +# hence accessing the "Transformation" attribute would throw an error) +times$Phase = factor(times$Phase, levels = c("Read", "Check", "Transformation", "Recheck")) + +times.wide = dcast(data = times, + formula = Tool + Workload + Description + Model + Run ~ Phase, + value.var = "Time", + drop = T, + fun.aggregate = mean +) + +# calculate aggregated values +times.derived = times.wide +times.derived$Read.and.Check = times.derived$Read + times.derived$Check +times.derived$Transformation.and.Recheck = times.derived$Transformation + times.derived$Recheck + +# calculate the median value of runs +times.aggregated.runs = ddply( + .data = times.derived, + .variables = c("Tool", "Workload", "Description", "Model"), + .fun = colwise(median), + .progress = "text" +) +# drop the "Run" column +times.aggregated.runs = subset(times.aggregated.runs, select = -c(Run)) + +times.processed = melt( + data = times.aggregated.runs, + id.vars = c("Tool", "Workload", "Description", "Model"), + measure.vars = phases, + variable.name = "Phase", + value.name = "Time" +) + +# beautify plotted record: +# 1. change dots to spaces +# 2. make sure that the phases are still factors +times.plot = times.processed +times.plot$Phase = gsub('\\.', ' ', times.plot$Phase) +times.plot$Phase = factor(times.plot$Phase, levels = phasesPrettified) +times.plot$Workload = factor(times.plot$Workload, levels = workloads) + +### line charts +for (phase in phasesPrettified) { + phase.filename = gsub(' ', '-', phase) + workloadSizes = sizes[["Repair"]] + + # filter the dataframe to the current phase + df = times.plot[times.plot$Phase == phase, ] + + # do not visualize empty data sets + if (nrow(df) == 0) { + print(paste("No rows to visualize for phase", phase)) + next + } + + # x axis labels + xbreaks = unique(df$Model) + currentWorkloadSizes = head(workloadSizes, n=length(xbreaks)) + xlabels = paste(xbreaks, "\n", currentWorkloadSizes, sep = "") + + # drop every other models size + maxLabel = max(log2(max(df$Model)), 2) + if (maxLabel %% 2) { + start = 3 + } else { + start = 2 + } + filter = seq(start, maxLabel, by=2) + + xlabels[filter] = "" + + # y axis labels + yaxis = nice_y_axis() + ybreaks = yaxis$ybreaks + ylabels = yaxis$ylabels + + p = ggplot(df) + #na.omit(df)) + + aes(x = as.factor(Model), y = Time) + + labs(title = paste("Individual query execution time,", phase, "phase, ", toolName), x = "Model size\n#Elements", y = "Execution times [ms]") + + geom_point(aes(col = Tool, shape = Tool), size = 2.0) + + scale_shape_manual(values = seq(0, 15)) + + geom_line(aes(col = Tool, group = Tool), size = 0.5) + + scale_x_discrete(breaks = xbreaks, labels = xlabels) + + scale_y_log10(breaks = ybreaks, labels = ylabels) + + guides(color = guide_legend(ncol = 4)) + + theme_bw() + + theme( + plot.title = element_text(hjust = 0.5), + text = element_text(size = 10), + legend.key = element_blank(), + legend.title = element_blank(), + legend.position = "bottom", + axis.text = element_text(size = 9) + ) + print(p) + for (cpage in 1:6) { + ggsave( + plot = p + facet_grid_paginate(~ Workload, nrow=1, ncol = 1, page=cpage, scale = "free"), + filename = paste(output_dir, "/", toolName, "-", phase.filename, "-",workloads[cpage], ".pdf", sep=""), + width = 250, height = 150, units = "mm" + ) + } +} diff --git a/trainbenchmark/trainbenchmark-reporting/toolwise.py b/trainbenchmark/trainbenchmark-reporting/toolwise.py new file mode 100644 index 0000000000000000000000000000000000000000..93a2753579af2ef62b9fd7a4cd3100f7d0cff208 --- /dev/null +++ b/trainbenchmark/trainbenchmark-reporting/toolwise.py @@ -0,0 +1,33 @@ +import argparse +import json +import logging +import os.path +import subprocess + + +FORMAT = '%(asctime)s %(levelname)-8s %(threadName)-10s (%(filename)s:%(lineno)d): %(message)s' +logger = logging.getLogger('toolwise') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Plot results per tool.') + parser.add_argument( + "-v", "--verbose", help="Print debug messages.", action="store_true") + parser.add_argument( + "-f", "--file-config", default='merge_results.json', help="Config file to use.") + args = parser.parse_args() + logging.basicConfig(format=FORMAT, level=logging.DEBUG if args.verbose else logging.INFO) + # load config file + with open('merge_results.json') as fdr: + content = json.load(fdr) + # update with local version, if existing + directory, basename = os.path.split(os.path.abspath('merge_results.json')) + local_config_file = os.path.join(directory, 'local-' + basename) + if os.path.exists(local_config_file): + with open(local_config_file) as fdr: + content.update(json.load(fdr)) + else: + logger.debug('No local config file found.') + for tool in content.get('toolwise', []): + logging.info('Processing %s now.', tool) + subprocess.call(["Rscript", "toolwise.R", tool]) diff --git a/trainbenchmark/trainbenchmark-reporting/toolwise.sh b/trainbenchmark/trainbenchmark-reporting/toolwise.sh new file mode 100755 index 0000000000000000000000000000000000000000..b9891085ed77d2fefa1f29149d65093e5607e0b9 --- /dev/null +++ b/trainbenchmark/trainbenchmark-reporting/toolwise.sh @@ -0,0 +1,4 @@ +#!/bin/bash +python toolwise.py +# --file-config <file> +# --verbose diff --git a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/config/BenchmarkConfigBase.java b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/config/BenchmarkConfigBase.java index 8467833190a8872e79dacdf59a40b8923cbeebb8..8ac9f8c43332d12e21fd930026cc3248119238d4 100644 --- a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/config/BenchmarkConfigBase.java +++ b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/config/BenchmarkConfigBase.java @@ -24,7 +24,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase { /** * The id for the benchmark, used for determining the result directory. */ - protected final int benchmarkId; + protected final String benchmarkId; /** * The timeout for each measurement run in seconds. @@ -68,7 +68,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase { */ protected final Optional<Integer> transformationConstant; - protected BenchmarkConfigBase(final int benchmarkId, final long timeout, final int runs, final String modelFilename, + protected BenchmarkConfigBase(final String benchmarkId, final long timeout, final int runs, final String modelFilename, final List<RailwayOperation> operations, final String workload, final TransformationChangeSetStrategy transformationChangeSetStrategy, final Optional<Integer> queryTransformationCount, final Optional<Integer> transformationConstant) { @@ -84,7 +84,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase { this.transformationConstant = transformationConstant; } - public int getBenchmarkId() { + public String getBenchmarkId() { return benchmarkId; } diff --git a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/config/BenchmarkConfigBaseBuilder.java b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/config/BenchmarkConfigBaseBuilder.java index 03fb69424a80bf657b84135f5034dba2f00c5194..6e5b1d23253344b852bdf14d7ed017b183182adc 100644 --- a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/config/BenchmarkConfigBaseBuilder.java +++ b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/config/BenchmarkConfigBaseBuilder.java @@ -8,7 +8,7 @@ import com.google.common.base.Preconditions; import hu.bme.mit.trainbenchmark.constants.RailwayOperation; public final class BenchmarkConfigBaseBuilder { - private Integer benchmarkId; + private String benchmarkId; private Long timeout; private Integer runs; private String modelFilename; @@ -65,7 +65,7 @@ public final class BenchmarkConfigBaseBuilder { return this; } - public BenchmarkConfigBaseBuilder setBenchmarkId(final Integer benchmarkId) { + public BenchmarkConfigBaseBuilder setBenchmarkId(final String benchmarkId) { this.benchmarkId = benchmarkId; return this; } diff --git a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/result/AbstractResult.java b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/result/AbstractResult.java index a8143f187ebb36a11ba38191840fdff32191b03d..f2a13294252520192b52c0b2a544adca7c009310 100644 --- a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/result/AbstractResult.java +++ b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/result/AbstractResult.java @@ -2,6 +2,7 @@ package hu.bme.mit.trainbenchmark.benchmark.result; import java.io.File; import java.io.IOException; +import java.nio.charset.Charset; import org.apache.commons.io.FileUtils; @@ -24,8 +25,9 @@ public abstract class AbstractResult { protected final String MODEL = "Model"; protected final String RUN = "Run"; - protected final int benchmarkId; + protected final String benchmarkId; protected final String toolName; + protected final String projectName; protected final String workload; protected final String workspaceDir; protected final String model; @@ -34,6 +36,7 @@ public abstract class AbstractResult { public AbstractResult(final BenchmarkConfig bc) { this.benchmarkId = bc.getConfigBase().getBenchmarkId(); this.toolName = bc.getToolName(); + this.projectName = bc.getProjectName(); this.workload = bc.getConfigBase().getWorkload(); this.workspaceDir = bc.getConfigBase().getWorkspaceDir(); this.model = bc.getConfigBase().getModelFilename(); @@ -41,13 +44,15 @@ public abstract class AbstractResult { } public void serializeCsv(final String csv, final String filePrefix) throws IOException { - final String matchesCsvPath = String.format("%s/%s-%s-%s-%s-%s.csv", getResultDir(), filePrefix, toolName, - workload, model, description); - FileUtils.write(new File(matchesCsvPath), csv); + File targetDir = ResultHelper.getResultDirectory(projectName, benchmarkId); + String fileName = String.format("%s-%s-%s-%s-%s.csv", filePrefix, toolName, workload, model, description); +// final String matchesCsvPath = String.format("%s/%s-%s-%s-%s-%s.csv", getResultDir(), filePrefix, toolName, +// workload, model, description); + FileUtils.write(FileUtils.getFile(targetDir, fileName), csv, Charset.defaultCharset()); } - public String getResultDir() { - return workspaceDir + ResultHelper.getResultDirForId(benchmarkId); - } +// public String getResultDir() { +// return workspaceDir + ResultHelper.getResultDirForId(benchmarkId); +// } } diff --git a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/result/ResultHelper.java b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/result/ResultHelper.java index 56ca0bbedf4cbe7ffd982a478514efb1aec758d2..737335677b1848f11ccc2e48259ffbeca4e9a694 100644 --- a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/result/ResultHelper.java +++ b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/result/ResultHelper.java @@ -3,8 +3,15 @@ package hu.bme.mit.trainbenchmark.benchmark.result; import java.io.File; import java.io.IOException; import java.nio.file.*; +import java.text.DateFormat; +import java.text.SimpleDateFormat; import java.util.Arrays; +import java.util.Collection; +import java.util.Date; +import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfig; +import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfigBuilder; +import hu.bme.mit.trainbenchmark.config.ExecutionConfig; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; @@ -12,6 +19,7 @@ public final class ResultHelper { public static final String WORKSPACE_DIR = "../"; public static final String RESULT_DIR = "results/"; + public static final String TOOLS_DIR = "tools/"; public static final String DIAGRAM_DIR = "diagrams/"; public static final String SCRIPT_PATH = WORKSPACE_DIR + "trainbenchmark-scripts/src/"; public static final String BENCHMARK_SCRIPT = "BenchmarkScript.groovy"; @@ -94,4 +102,38 @@ public final class ResultHelper { FileUtils.copyFile(srcFile, destFile); } + public static void createNewResultDirs(Collection<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> tools) { + } + + /** + * Prepare a benchmark run, i.e., creates necessary directories. This operation is idempotent. + * @param config the benchmark configuration + * @param ec the execution configuration + */ + public static void prepare(BenchmarkConfig config, ExecutionConfig ec) { + // ensure directory for the tool + File toolDir = getToolDirectory(config.getProjectName()); + toolDir.mkdir(); + // ensure directory for the run inside toolDir + File runDir = getResultDirectory(toolDir, config.getConfigBase().getBenchmarkId()); + runDir.mkdir(); + } + + public static String getNow() { + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss"); + return dateFormat.format(new Date()); + } + + public static File getToolDirectory(String projectName) { + return FileUtils.getFile(WORKSPACE_DIR, RESULT_DIR, TOOLS_DIR, projectName); + } + + public static File getResultDirectory(File toolDirectory, String benchmarkId) { + return FileUtils.getFile(toolDirectory, "run-" + benchmarkId); + } + + public static File getResultDirectory(String projectName, String benchmarkId) { + return getResultDirectory(getToolDirectory(projectName), benchmarkId); + } + } diff --git a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/runcomponents/BenchmarkRunner.java b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/runcomponents/BenchmarkRunner.java index 9136edcca823a519a228c863d31ec4b1aaada6c4..06d8bb80862e12e22e66e9c30b3742560367c0b4 100644 --- a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/runcomponents/BenchmarkRunner.java +++ b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/runcomponents/BenchmarkRunner.java @@ -11,14 +11,38 @@ import org.apache.commons.exec.Executor; import org.apache.commons.exec.PumpStreamHandler; import com.google.common.base.Joiner; - +import com.google.common.collect.ImmutableMap; import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfig; import hu.bme.mit.trainbenchmark.benchmark.memory.MemoryResult; import hu.bme.mit.trainbenchmark.config.ExecutionConfig; +import org.apache.commons.exec.*; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Map; public class BenchmarkRunner { - public static int runPerformanceBenchmark(final BenchmarkConfig bc, final ExecutionConfig ec) + public static String dictFormat(String format, Map<String, Object> values) { + StringBuilder convFormat = new StringBuilder(format); + ArrayList<Object> valueList = new ArrayList<>(); + int currentPos = 1; + for (String key : values.keySet()) { + String formatKey = "%(" + key + ")", + formatPos = "%" + Integer.toString(currentPos) + "$1s"; + int index = -1; + while ((index = convFormat.indexOf(formatKey, index)) != -1) { + convFormat.replace(index, index + formatKey.length(), formatPos); + index += formatPos.length(); + } + valueList.add(values.get(key)); + ++currentPos; + } + return String.format(convFormat.toString(), valueList.toArray()); + } + + public static int runPerformanceBenchmark(final BenchmarkConfig bc, final ExecutionConfig ec, String jvmSetting) throws IOException, InterruptedException { final Joiner joiner = Joiner.on(", "); System.out.println("Running benchmark."); @@ -38,9 +62,17 @@ public class BenchmarkRunner { final String jarPath = String.format("../%s/build/libs/%s-1.0.0-SNAPSHOT-fat.jar %s", projectName, projectName, configPath); - final String javaCommand = String.format("java -Xms%s -Xmx%s -server -Xverify:none -jar %s %s", ec.getXms(), ec.getXmx(), jarPath, configPath); + final String javaCommand = dictFormat("java " + jvmSetting + " -jar %(jarPath) %(configPath)", + ImmutableMap.of("Xms", ec.getXms(), "Xmx", ec.getXmx(), + "jarPath", jarPath, "configPath", configPath)); + +// final String javaCommand = String.format("java -Xms%s -Xmx%s -server -Xverify:none -jar %s %s", ec.getXms(), ec.getXmx(), jarPath, configPath); // final String javaCommand = String.format("java -Xms%s -Xmx%s -server -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:StartFlightRecording=dumponexit=true -Xverify:none -jar %s %s", ec.getXms(), ec.getXmx(), jarPath, configPath); final CommandLine cmdLine = CommandLine.parse(javaCommand); + if (ec.getDryRun()) { + System.out.println("Would have executed: "+ cmdLine.toString()); + return 0; + } final long timeoutInSeconds = bc.getConfigBase().getTimeout(); final long timeoutInMilliseconds = timeoutInSeconds * 1000; @@ -63,7 +95,7 @@ public class BenchmarkRunner { } public static int runMemoryBenchmark(final BenchmarkConfig bc, final ExecutionConfig defaultEc, - final int numberOfSteps) throws IOException, InterruptedException { + final int numberOfSteps, String jvmSetting) throws IOException, InterruptedException { // e.g. initialMaxMemory = 12800, we save this (as a final variable), so // that we will not exceed it // @@ -79,7 +111,7 @@ public class BenchmarkRunner { memoryQuantum /= 2; final ExecutionConfig ec = new ExecutionConfig(currentMaxMemory, currentMaxMemory); - if (runPerformanceBenchmark(bc, ec) == 0) { + if (runPerformanceBenchmark(bc, ec, jvmSetting) == 0) { System.out.println("Execution finished, testing with less memory."); System.out.println(); currentMaxMemory -= memoryQuantum; diff --git a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/test/TrainBenchmarkTest.java b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/test/TrainBenchmarkTest.java index da4721781f6fffb5136e7780d2cd41f6da93c6da..d8ac00dd5b0e3464cae1cc1d919e46ad59bca0eb 100644 --- a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/test/TrainBenchmarkTest.java +++ b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/test/TrainBenchmarkTest.java @@ -23,7 +23,7 @@ public abstract class TrainBenchmarkTest { @Rule public ErrorCollector collector = new ErrorCollector(); - protected final int benchmarkId = 0; + protected final String benchmarkId = ""; protected ExecutionConfig executionConfig = ExecutionConfig.defaultExecutionConfig(); protected final long timeout = 120; protected final int runs = 1; diff --git a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/test/queryspecific/QueryTest.java b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/test/queryspecific/QueryTest.java index 4046712c480f14d106115bbea029fe14539d997b..f616b63636e4b2c5cd8a827177fb43b092a29da7 100644 --- a/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/test/queryspecific/QueryTest.java +++ b/trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/test/queryspecific/QueryTest.java @@ -11,7 +11,7 @@ import hu.bme.mit.trainbenchmark.config.ExecutionConfig; public abstract class QueryTest { - protected final int benchmarkId = 0; + protected final String benchmarkId = ""; protected ExecutionConfig executionConfig = ExecutionConfig.defaultExecutionConfig(); protected final long timeout = 120; protected final int runs = 1;