Merge remote-tracking branch 'sle/artifact-evaluation'

trainbenchmark/trainbenchmark-reporting/toolwise.R

+library(data.table)
+library(reshape2)
+library(plyr)
+library(ggplot2)
+library(ggrepel)
+library(arules)
+library(ggforce)
+
+source('util.R')
+
+args = commandArgs(trailingOnly=TRUE)
+if (length(args)==0) {
+  stop("At least one argument must be supplied (tool-name).\n", call.=FALSE)
+}
+toolName = args[1]
+
+# prepare output directory
+output_dir = paste("../diagrams/merged", toolName, sep="/")
+if (!(dir.exists(output_dir))) {
+    dir.create(output_dir)
+}
+
+# constants
+workloads = c(
+  "PosLength",  "SwitchMonitored",
+  "RouteSensor", "SwitchSet",
+  "ConnectedSegments", "SemaphoreNeighbor"
+)
+phases = c("Read", "Check", "Read.and.Check", "Transformation", "Recheck", "Transformation.and.Recheck")
+phasesPrettified = c("Read", "Check", "Read and Check", "Transformation", "Recheck", "Transformation and Recheck")
+
+sizes = list()      # 1     2      4      8      16      32      64      128     256     512     1024    2048   4096
+sizes[["Repair"]] = c("8k", "15k", "33k", "66k", "135k", "271k", "566k", "1.1M", "2.2M", "4.6M", "9.3M", "18M", "37M")
+
+runList = read.csv(paste("../results/tools/", toolName, "/run-list.csv", sep=""), colClasses=c(rep("character",1)))
+
+# load the data
+tsvs = list.files(paste("../results/tools/", toolName, "/all-runs/", sep=""), pattern = "times-.*\\.csv", full.names = T, recursive = T)
+
+l = lapply(tsvs, read.csv)
+times = rbindlist(l)
+
+# preprocess the data
+times$Tool = factor(times$Tool, levels = runList$Runs)
+keep_descriptions_first_char(times)
+
+times$Model = gsub("\\D+", "", times$Model)
+times$Model = as.numeric(times$Model)
+times$Time = times$Time / 10^6
+# make the phases a factor with a fixed set of values to help dcasting
+# (e.g. Batch measurements do not have Transformation and Recheck attributes,
+# hence accessing the "Transformation" attribute would throw an error)
+times$Phase = factor(times$Phase, levels = c("Read", "Check", "Transformation", "Recheck"))
+
+times.wide = dcast(data = times,
+                   formula = Tool + Workload + Description + Model + Run ~ Phase,
+                   value.var = "Time",
+                   drop = T,
+                   fun.aggregate = mean
+)
+
+# calculate aggregated values
+times.derived = times.wide
+times.derived$Read.and.Check = times.derived$Read + times.derived$Check
+times.derived$Transformation.and.Recheck = times.derived$Transformation + times.derived$Recheck
+
+# calculate the median value of runs
+times.aggregated.runs = ddply(
+  .data = times.derived,
+  .variables = c("Tool", "Workload", "Description", "Model"),
+  .fun = colwise(median),
+  .progress = "text"
+)
+# drop the "Run" column
+times.aggregated.runs = subset(times.aggregated.runs, select = -c(Run))
+
+times.processed = melt(
+  data = times.aggregated.runs,
+  id.vars = c("Tool", "Workload", "Description", "Model"),
+  measure.vars = phases,
+  variable.name = "Phase",
+  value.name = "Time"
+)
+
+# beautify plotted record:
+# 1. change dots to spaces
+# 2. make sure that the phases are still factors
+times.plot = times.processed
+times.plot$Phase = gsub('\\.', ' ', times.plot$Phase)
+times.plot$Phase = factor(times.plot$Phase, levels = phasesPrettified)
+times.plot$Workload = factor(times.plot$Workload, levels = workloads)
+
+### line charts
+for (phase in phasesPrettified) {
+  phase.filename = gsub(' ', '-', phase)
+  workloadSizes = sizes[["Repair"]]
+
+  # filter the dataframe to the current phase
+  df = times.plot[times.plot$Phase == phase, ]
+
+  # do not visualize empty data sets
+  if (nrow(df) == 0) {
+    print(paste("No rows to visualize for phase", phase))
+    next
+  }
+
+  # x axis labels
+  xbreaks = unique(df$Model)
+  currentWorkloadSizes = head(workloadSizes, n=length(xbreaks))
+  xlabels = paste(xbreaks, "\n", currentWorkloadSizes, sep = "")
+
+  # drop every other models size
+  maxLabel = max(log2(max(df$Model)), 2)
+  if (maxLabel %% 2) {
+    start = 3
+  } else {
+    start = 2
+  }
+  filter = seq(start, maxLabel, by=2)
+
+  xlabels[filter] = ""
+
+  # y axis labels
+  yaxis = nice_y_axis()
+  ybreaks = yaxis$ybreaks
+  ylabels = yaxis$ylabels
+
+  p = ggplot(df) + #na.omit(df)) +
+    aes(x = as.factor(Model), y = Time) +
+    labs(title = paste("Individual query execution time,", phase, "phase, ", toolName), x = "Model size\n#Elements", y = "Execution times [ms]") +
+    geom_point(aes(col = Tool, shape = Tool), size = 2.0) +
+    scale_shape_manual(values = seq(0, 15)) +
+    geom_line(aes(col = Tool, group = Tool), size = 0.5) +
+    scale_x_discrete(breaks = xbreaks, labels = xlabels) +
+    scale_y_log10(breaks = ybreaks, labels = ylabels) +
+    guides(color = guide_legend(ncol = 4)) +
+    theme_bw() +
+    theme(
+      plot.title = element_text(hjust = 0.5),
+      text = element_text(size = 10),
+      legend.key = element_blank(),
+      legend.title = element_blank(),
+      legend.position = "bottom",
+      axis.text = element_text(size = 9)
+    )
+  print(p)
+  for (cpage in 1:6) {
+    ggsave(
+      plot = p + facet_grid_paginate(~ Workload, nrow=1, ncol = 1, page=cpage, scale = "free"),
+      filename = paste(output_dir, "/", toolName, "-", phase.filename, "-",workloads[cpage], ".pdf", sep=""),
+      width = 250, height = 150, units = "mm"
+    )
+  }
+}

trainbenchmark/trainbenchmark-reporting/toolwise.py

+import argparse
+import json
+import logging
+import os.path
+import subprocess
+
+
+FORMAT = '%(asctime)s %(levelname)-8s %(threadName)-10s (%(filename)s:%(lineno)d): %(message)s'
+logger = logging.getLogger('toolwise')
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Plot results per tool.')
+    parser.add_argument(
+        "-v", "--verbose", help="Print debug messages.", action="store_true")
+    parser.add_argument(
+        "-f", "--file-config", default='merge_results.json', help="Config file to use.")
+    args = parser.parse_args()
+    logging.basicConfig(format=FORMAT, level=logging.DEBUG if args.verbose else logging.INFO)
+    # load config file
+    with open('merge_results.json') as fdr:
+        content = json.load(fdr)
+    # update with local version, if existing
+    directory, basename = os.path.split(os.path.abspath('merge_results.json'))
+    local_config_file = os.path.join(directory, 'local-' + basename)
+    if os.path.exists(local_config_file):
+        with open(local_config_file) as fdr:
+            content.update(json.load(fdr))
+    else:
+        logger.debug('No local config file found.')
+    for tool in content.get('toolwise', []):
+        logging.info('Processing %s now.', tool)
+        subprocess.call(["Rscript", "toolwise.R", tool])

trainbenchmark/trainbenchmark-reporting/toolwise.sh

+#!/bin/bash
+python toolwise.py
+# --file-config <file>
+# --verbose

trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/config/BenchmarkConfigBase.java

@@ -24,7 +24,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase {
 	/**
 	 * The id for the benchmark, used for determining the result directory.
 	 */
-	protected final int benchmarkId;
+	protected final String benchmarkId;
 
 	/**
 	 * The timeout for each measurement run in seconds.
@@ -68,7 +68,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase {
 	 */
 	protected final Optional<Integer> transformationConstant;
 
-	protected BenchmarkConfigBase(final int benchmarkId, final long timeout, final int runs, final String modelFilename,
+	protected BenchmarkConfigBase(final String benchmarkId, final long timeout, final int runs, final String modelFilename,
 			final List<RailwayOperation> operations, final String workload,
 			final TransformationChangeSetStrategy transformationChangeSetStrategy,
 			final Optional<Integer> queryTransformationCount, final Optional<Integer> transformationConstant) {
@@ -84,7 +84,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase {
 		this.transformationConstant = transformationConstant;
 	}
 
-	public int getBenchmarkId() {
+	public String getBenchmarkId() {
 		return benchmarkId;
 	}
 

trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/config/BenchmarkConfigBaseBuilder.java

@@ -8,7 +8,7 @@ import com.google.common.base.Preconditions;
 import hu.bme.mit.trainbenchmark.constants.RailwayOperation;
 
 public final class BenchmarkConfigBaseBuilder {
-	private Integer benchmarkId;
+	private String benchmarkId;
 	private Long timeout;
 	private Integer runs;
 	private String modelFilename;
@@ -65,7 +65,7 @@ public final class BenchmarkConfigBaseBuilder {
 		return this;
 	}
 
-	public BenchmarkConfigBaseBuilder setBenchmarkId(final Integer benchmarkId) {
+	public BenchmarkConfigBaseBuilder setBenchmarkId(final String benchmarkId) {
 		this.benchmarkId = benchmarkId;
 		return this;
 	}

trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/result/AbstractResult.java

@@ -2,6 +2,7 @@ package hu.bme.mit.trainbenchmark.benchmark.result;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.charset.Charset;
 
 import org.apache.commons.io.FileUtils;
 
@@ -24,8 +25,9 @@ public abstract class AbstractResult {
 	protected final String MODEL = "Model";
 	protected final String RUN = "Run";
 
-	protected final int benchmarkId;
+	protected final String benchmarkId;
 	protected final String toolName;
+	protected final String projectName;
 	protected final String workload;
 	protected final String workspaceDir;
 	protected final String model;
@@ -34,6 +36,7 @@ public abstract class AbstractResult {
 	public AbstractResult(final BenchmarkConfig bc) {
 		this.benchmarkId = bc.getConfigBase().getBenchmarkId();
 		this.toolName = bc.getToolName();
+		this.projectName = bc.getProjectName();
 		this.workload = bc.getConfigBase().getWorkload();
 		this.workspaceDir = bc.getConfigBase().getWorkspaceDir();
 		this.model = bc.getConfigBase().getModelFilename();
@@ -41,13 +44,15 @@ public abstract class AbstractResult {
 	}
 
 	public void serializeCsv(final String csv, final String filePrefix) throws IOException {
-		final String matchesCsvPath = String.format("%s/%s-%s-%s-%s-%s.csv", getResultDir(), filePrefix, toolName,
-				workload, model, description);
-		FileUtils.write(new File(matchesCsvPath), csv);
+		File targetDir = ResultHelper.getResultDirectory(projectName, benchmarkId);
+		String fileName = String.format("%s-%s-%s-%s-%s.csv", filePrefix, toolName, workload, model, description);
+//		final String matchesCsvPath = String.format("%s/%s-%s-%s-%s-%s.csv", getResultDir(), filePrefix, toolName,
+//				workload, model, description);
+		FileUtils.write(FileUtils.getFile(targetDir, fileName), csv, Charset.defaultCharset());
 	}
 
-	public String getResultDir() {
-		return workspaceDir + ResultHelper.getResultDirForId(benchmarkId);
-	}
+//	public String getResultDir() {
+//		return workspaceDir + ResultHelper.getResultDirForId(benchmarkId);
+//	}
 
 }

trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/result/ResultHelper.java

@@ -3,8 +3,15 @@ package hu.bme.mit.trainbenchmark.benchmark.result;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.*;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
 import java.util.Arrays;
+import java.util.Collection;
+import java.util.Date;
 
+import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfig;
+import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfigBuilder;
+import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.StringUtils;
 
@@ -12,6 +19,7 @@ public final class ResultHelper {
 
 	public static final String WORKSPACE_DIR = "../";
 	public static final String RESULT_DIR = "results/";
+	public static final String TOOLS_DIR = "tools/";
 	public static final String DIAGRAM_DIR = "diagrams/";
 	public static final String SCRIPT_PATH = WORKSPACE_DIR + "trainbenchmark-scripts/src/";
 	public static final String BENCHMARK_SCRIPT = "BenchmarkScript.groovy";
@@ -94,4 +102,38 @@ public final class ResultHelper {
 		FileUtils.copyFile(srcFile, destFile);
 	}
 
+	public static void createNewResultDirs(Collection<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> tools) {
+	}
+
+	/**
+	 * Prepare a benchmark run, i.e., creates necessary directories. This operation is idempotent.
+	 * @param config the benchmark configuration
+	 * @param ec the execution configuration
+	 */
+	public static void prepare(BenchmarkConfig config, ExecutionConfig ec) {
+		// ensure directory for the tool
+		File toolDir = getToolDirectory(config.getProjectName());
+		toolDir.mkdir();
+		// ensure directory for the run inside toolDir
+		File runDir = getResultDirectory(toolDir, config.getConfigBase().getBenchmarkId());
+		runDir.mkdir();
+	}
+
+	public static String getNow() {
+		DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
+		return dateFormat.format(new Date());
+	}
+
+	public static File getToolDirectory(String projectName) {
+		return FileUtils.getFile(WORKSPACE_DIR, RESULT_DIR, TOOLS_DIR, projectName);
+	}
+
+	public static File getResultDirectory(File toolDirectory, String benchmarkId) {
+		return FileUtils.getFile(toolDirectory, "run-" + benchmarkId);
+	}
+
+	public static File getResultDirectory(String projectName, String benchmarkId) {
+		return getResultDirectory(getToolDirectory(projectName), benchmarkId);
+	}
+
 }

trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/runcomponents/BenchmarkRunner.java

@@ -11,14 +11,38 @@ import org.apache.commons.exec.Executor;
 import org.apache.commons.exec.PumpStreamHandler;
 
 import com.google.common.base.Joiner;
-
+import com.google.common.collect.ImmutableMap;
 import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfig;
 import hu.bme.mit.trainbenchmark.benchmark.memory.MemoryResult;
 import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
+import org.apache.commons.exec.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Map;
 
 public class BenchmarkRunner {
 
-	public static int runPerformanceBenchmark(final BenchmarkConfig bc, final ExecutionConfig ec)
+	public static String dictFormat(String format, Map<String, Object> values) {
+		StringBuilder convFormat = new StringBuilder(format);
+		ArrayList<Object> valueList = new ArrayList<>();
+		int currentPos = 1;
+		for (String key : values.keySet()) {
+			String formatKey = "%(" + key + ")",
+					formatPos = "%" + Integer.toString(currentPos) + "$1s";
+			int index = -1;
+			while ((index = convFormat.indexOf(formatKey, index)) != -1) {
+				convFormat.replace(index, index + formatKey.length(), formatPos);
+				index += formatPos.length();
+			}
+			valueList.add(values.get(key));
+			++currentPos;
+		}
+		return String.format(convFormat.toString(), valueList.toArray());
+	}
+
+	public static int runPerformanceBenchmark(final BenchmarkConfig bc, final ExecutionConfig ec, String jvmSetting)
 			throws IOException, InterruptedException {
 		final Joiner joiner = Joiner.on(", ");
 		System.out.println("Running benchmark.");
@@ -38,9 +62,17 @@ public class BenchmarkRunner {
 		final String jarPath = String.format("../%s/build/libs/%s-1.0.0-SNAPSHOT-fat.jar %s", projectName, projectName,
 				configPath);
 
-		final String javaCommand = String.format("java -Xms%s -Xmx%s -server -Xverify:none -jar %s %s", ec.getXms(), ec.getXmx(), jarPath, configPath);
+		final String javaCommand = dictFormat("java " + jvmSetting + " -jar %(jarPath) %(configPath)",
+				ImmutableMap.of("Xms", ec.getXms(), "Xmx", ec.getXmx(),
+						"jarPath", jarPath, "configPath", configPath));
+
+//		final String javaCommand = String.format("java -Xms%s -Xmx%s -server -Xverify:none -jar %s %s", ec.getXms(), ec.getXmx(), jarPath, configPath);
 //		final String javaCommand = String.format("java -Xms%s -Xmx%s -server -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:StartFlightRecording=dumponexit=true -Xverify:none -jar %s %s", ec.getXms(), ec.getXmx(), jarPath, configPath);
 		final CommandLine cmdLine = CommandLine.parse(javaCommand);
+		if (ec.getDryRun()) {
+			System.out.println("Would have executed: "+ cmdLine.toString());
+			return 0;
+		}
 
 		final long timeoutInSeconds = bc.getConfigBase().getTimeout();
 		final long timeoutInMilliseconds = timeoutInSeconds * 1000;
@@ -63,7 +95,7 @@ public class BenchmarkRunner {
 	}
 
 	public static int runMemoryBenchmark(final BenchmarkConfig bc, final ExecutionConfig defaultEc,
-			final int numberOfSteps) throws IOException, InterruptedException {
+			final int numberOfSteps, String jvmSetting) throws IOException, InterruptedException {
 		// e.g. initialMaxMemory = 12800, we save this (as a final variable), so
 		// that we will not exceed it
 		//
@@ -79,7 +111,7 @@ public class BenchmarkRunner {
 			memoryQuantum /= 2;
 
 			final ExecutionConfig ec = new ExecutionConfig(currentMaxMemory, currentMaxMemory);
-			if (runPerformanceBenchmark(bc, ec) == 0) {
+			if (runPerformanceBenchmark(bc, ec, jvmSetting) == 0) {
 				System.out.println("Execution finished, testing with less memory.");
 				System.out.println();
 				currentMaxMemory -= memoryQuantum;

trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/test/TrainBenchmarkTest.java

@@ -23,7 +23,7 @@ public abstract class TrainBenchmarkTest {
 	@Rule
 	public ErrorCollector collector = new ErrorCollector();
 
-	protected final int benchmarkId = 0;
+	protected final String benchmarkId = "";
 	protected ExecutionConfig executionConfig = ExecutionConfig.defaultExecutionConfig();
 	protected final long timeout = 120;
 	protected final int runs = 1;

trainbenchmark/trainbenchmark-tool/src/main/java/hu/bme/mit/trainbenchmark/benchmark/test/queryspecific/QueryTest.java

@@ -11,7 +11,7 @@ import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
 
 public abstract class QueryTest {
 
-	protected final int benchmarkId = 0;
+	protected final String benchmarkId = "";
 	protected ExecutionConfig executionConfig = ExecutionConfig.defaultExecutionConfig();
 	protected final long timeout = 120;
 	protected final int runs = 1;