Skip to content
Snippets Groups Projects
Commit 1b6cad29 authored by Johannes Mey's avatar Johannes Mey
Browse files

Merge remote-tracking branch 'sle/artifact-evaluation'

parents 5594f6b7 dcea58ae
No related branches found
No related tags found
No related merge requests found
Showing
with 289 additions and 19 deletions
library(data.table)
library(reshape2)
library(plyr)
library(ggplot2)
library(ggrepel)
library(arules)
library(ggforce)
source('util.R')
args = commandArgs(trailingOnly=TRUE)
if (length(args)==0) {
stop("At least one argument must be supplied (tool-name).\n", call.=FALSE)
}
toolName = args[1]
# prepare output directory
output_dir = paste("../diagrams/merged", toolName, sep="/")
if (!(dir.exists(output_dir))) {
dir.create(output_dir)
}
# constants
workloads = c(
"PosLength", "SwitchMonitored",
"RouteSensor", "SwitchSet",
"ConnectedSegments", "SemaphoreNeighbor"
)
phases = c("Read", "Check", "Read.and.Check", "Transformation", "Recheck", "Transformation.and.Recheck")
phasesPrettified = c("Read", "Check", "Read and Check", "Transformation", "Recheck", "Transformation and Recheck")
sizes = list() # 1 2 4 8 16 32 64 128 256 512 1024 2048 4096
sizes[["Repair"]] = c("8k", "15k", "33k", "66k", "135k", "271k", "566k", "1.1M", "2.2M", "4.6M", "9.3M", "18M", "37M")
runList = read.csv(paste("../results/tools/", toolName, "/run-list.csv", sep=""), colClasses=c(rep("character",1)))
# load the data
tsvs = list.files(paste("../results/tools/", toolName, "/all-runs/", sep=""), pattern = "times-.*\\.csv", full.names = T, recursive = T)
l = lapply(tsvs, read.csv)
times = rbindlist(l)
# preprocess the data
times$Tool = factor(times$Tool, levels = runList$Runs)
keep_descriptions_first_char(times)
times$Model = gsub("\\D+", "", times$Model)
times$Model = as.numeric(times$Model)
times$Time = times$Time / 10^6
# make the phases a factor with a fixed set of values to help dcasting
# (e.g. Batch measurements do not have Transformation and Recheck attributes,
# hence accessing the "Transformation" attribute would throw an error)
times$Phase = factor(times$Phase, levels = c("Read", "Check", "Transformation", "Recheck"))
times.wide = dcast(data = times,
formula = Tool + Workload + Description + Model + Run ~ Phase,
value.var = "Time",
drop = T,
fun.aggregate = mean
)
# calculate aggregated values
times.derived = times.wide
times.derived$Read.and.Check = times.derived$Read + times.derived$Check
times.derived$Transformation.and.Recheck = times.derived$Transformation + times.derived$Recheck
# calculate the median value of runs
times.aggregated.runs = ddply(
.data = times.derived,
.variables = c("Tool", "Workload", "Description", "Model"),
.fun = colwise(median),
.progress = "text"
)
# drop the "Run" column
times.aggregated.runs = subset(times.aggregated.runs, select = -c(Run))
times.processed = melt(
data = times.aggregated.runs,
id.vars = c("Tool", "Workload", "Description", "Model"),
measure.vars = phases,
variable.name = "Phase",
value.name = "Time"
)
# beautify plotted record:
# 1. change dots to spaces
# 2. make sure that the phases are still factors
times.plot = times.processed
times.plot$Phase = gsub('\\.', ' ', times.plot$Phase)
times.plot$Phase = factor(times.plot$Phase, levels = phasesPrettified)
times.plot$Workload = factor(times.plot$Workload, levels = workloads)
### line charts
for (phase in phasesPrettified) {
phase.filename = gsub(' ', '-', phase)
workloadSizes = sizes[["Repair"]]
# filter the dataframe to the current phase
df = times.plot[times.plot$Phase == phase, ]
# do not visualize empty data sets
if (nrow(df) == 0) {
print(paste("No rows to visualize for phase", phase))
next
}
# x axis labels
xbreaks = unique(df$Model)
currentWorkloadSizes = head(workloadSizes, n=length(xbreaks))
xlabels = paste(xbreaks, "\n", currentWorkloadSizes, sep = "")
# drop every other models size
maxLabel = max(log2(max(df$Model)), 2)
if (maxLabel %% 2) {
start = 3
} else {
start = 2
}
filter = seq(start, maxLabel, by=2)
xlabels[filter] = ""
# y axis labels
yaxis = nice_y_axis()
ybreaks = yaxis$ybreaks
ylabels = yaxis$ylabels
p = ggplot(df) + #na.omit(df)) +
aes(x = as.factor(Model), y = Time) +
labs(title = paste("Individual query execution time,", phase, "phase, ", toolName), x = "Model size\n#Elements", y = "Execution times [ms]") +
geom_point(aes(col = Tool, shape = Tool), size = 2.0) +
scale_shape_manual(values = seq(0, 15)) +
geom_line(aes(col = Tool, group = Tool), size = 0.5) +
scale_x_discrete(breaks = xbreaks, labels = xlabels) +
scale_y_log10(breaks = ybreaks, labels = ylabels) +
guides(color = guide_legend(ncol = 4)) +
theme_bw() +
theme(
plot.title = element_text(hjust = 0.5),
text = element_text(size = 10),
legend.key = element_blank(),
legend.title = element_blank(),
legend.position = "bottom",
axis.text = element_text(size = 9)
)
print(p)
for (cpage in 1:6) {
ggsave(
plot = p + facet_grid_paginate(~ Workload, nrow=1, ncol = 1, page=cpage, scale = "free"),
filename = paste(output_dir, "/", toolName, "-", phase.filename, "-",workloads[cpage], ".pdf", sep=""),
width = 250, height = 150, units = "mm"
)
}
}
import argparse
import json
import logging
import os.path
import subprocess
FORMAT = '%(asctime)s %(levelname)-8s %(threadName)-10s (%(filename)s:%(lineno)d): %(message)s'
logger = logging.getLogger('toolwise')
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Plot results per tool.')
parser.add_argument(
"-v", "--verbose", help="Print debug messages.", action="store_true")
parser.add_argument(
"-f", "--file-config", default='merge_results.json', help="Config file to use.")
args = parser.parse_args()
logging.basicConfig(format=FORMAT, level=logging.DEBUG if args.verbose else logging.INFO)
# load config file
with open('merge_results.json') as fdr:
content = json.load(fdr)
# update with local version, if existing
directory, basename = os.path.split(os.path.abspath('merge_results.json'))
local_config_file = os.path.join(directory, 'local-' + basename)
if os.path.exists(local_config_file):
with open(local_config_file) as fdr:
content.update(json.load(fdr))
else:
logger.debug('No local config file found.')
for tool in content.get('toolwise', []):
logging.info('Processing %s now.', tool)
subprocess.call(["Rscript", "toolwise.R", tool])
#!/bin/bash
python toolwise.py
# --file-config <file>
# --verbose
...@@ -24,7 +24,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase { ...@@ -24,7 +24,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase {
/** /**
* The id for the benchmark, used for determining the result directory. * The id for the benchmark, used for determining the result directory.
*/ */
protected final int benchmarkId; protected final String benchmarkId;
/** /**
* The timeout for each measurement run in seconds. * The timeout for each measurement run in seconds.
...@@ -68,7 +68,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase { ...@@ -68,7 +68,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase {
*/ */
protected final Optional<Integer> transformationConstant; protected final Optional<Integer> transformationConstant;
protected BenchmarkConfigBase(final int benchmarkId, final long timeout, final int runs, final String modelFilename, protected BenchmarkConfigBase(final String benchmarkId, final long timeout, final int runs, final String modelFilename,
final List<RailwayOperation> operations, final String workload, final List<RailwayOperation> operations, final String workload,
final TransformationChangeSetStrategy transformationChangeSetStrategy, final TransformationChangeSetStrategy transformationChangeSetStrategy,
final Optional<Integer> queryTransformationCount, final Optional<Integer> transformationConstant) { final Optional<Integer> queryTransformationCount, final Optional<Integer> transformationConstant) {
...@@ -84,7 +84,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase { ...@@ -84,7 +84,7 @@ public final class BenchmarkConfigBase extends AbstractConfigBase {
this.transformationConstant = transformationConstant; this.transformationConstant = transformationConstant;
} }
public int getBenchmarkId() { public String getBenchmarkId() {
return benchmarkId; return benchmarkId;
} }
......
...@@ -8,7 +8,7 @@ import com.google.common.base.Preconditions; ...@@ -8,7 +8,7 @@ import com.google.common.base.Preconditions;
import hu.bme.mit.trainbenchmark.constants.RailwayOperation; import hu.bme.mit.trainbenchmark.constants.RailwayOperation;
public final class BenchmarkConfigBaseBuilder { public final class BenchmarkConfigBaseBuilder {
private Integer benchmarkId; private String benchmarkId;
private Long timeout; private Long timeout;
private Integer runs; private Integer runs;
private String modelFilename; private String modelFilename;
...@@ -65,7 +65,7 @@ public final class BenchmarkConfigBaseBuilder { ...@@ -65,7 +65,7 @@ public final class BenchmarkConfigBaseBuilder {
return this; return this;
} }
public BenchmarkConfigBaseBuilder setBenchmarkId(final Integer benchmarkId) { public BenchmarkConfigBaseBuilder setBenchmarkId(final String benchmarkId) {
this.benchmarkId = benchmarkId; this.benchmarkId = benchmarkId;
return this; return this;
} }
......
...@@ -2,6 +2,7 @@ package hu.bme.mit.trainbenchmark.benchmark.result; ...@@ -2,6 +2,7 @@ package hu.bme.mit.trainbenchmark.benchmark.result;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.Charset;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
...@@ -24,8 +25,9 @@ public abstract class AbstractResult { ...@@ -24,8 +25,9 @@ public abstract class AbstractResult {
protected final String MODEL = "Model"; protected final String MODEL = "Model";
protected final String RUN = "Run"; protected final String RUN = "Run";
protected final int benchmarkId; protected final String benchmarkId;
protected final String toolName; protected final String toolName;
protected final String projectName;
protected final String workload; protected final String workload;
protected final String workspaceDir; protected final String workspaceDir;
protected final String model; protected final String model;
...@@ -34,6 +36,7 @@ public abstract class AbstractResult { ...@@ -34,6 +36,7 @@ public abstract class AbstractResult {
public AbstractResult(final BenchmarkConfig bc) { public AbstractResult(final BenchmarkConfig bc) {
this.benchmarkId = bc.getConfigBase().getBenchmarkId(); this.benchmarkId = bc.getConfigBase().getBenchmarkId();
this.toolName = bc.getToolName(); this.toolName = bc.getToolName();
this.projectName = bc.getProjectName();
this.workload = bc.getConfigBase().getWorkload(); this.workload = bc.getConfigBase().getWorkload();
this.workspaceDir = bc.getConfigBase().getWorkspaceDir(); this.workspaceDir = bc.getConfigBase().getWorkspaceDir();
this.model = bc.getConfigBase().getModelFilename(); this.model = bc.getConfigBase().getModelFilename();
...@@ -41,13 +44,15 @@ public abstract class AbstractResult { ...@@ -41,13 +44,15 @@ public abstract class AbstractResult {
} }
public void serializeCsv(final String csv, final String filePrefix) throws IOException { public void serializeCsv(final String csv, final String filePrefix) throws IOException {
final String matchesCsvPath = String.format("%s/%s-%s-%s-%s-%s.csv", getResultDir(), filePrefix, toolName, File targetDir = ResultHelper.getResultDirectory(projectName, benchmarkId);
workload, model, description); String fileName = String.format("%s-%s-%s-%s-%s.csv", filePrefix, toolName, workload, model, description);
FileUtils.write(new File(matchesCsvPath), csv); // final String matchesCsvPath = String.format("%s/%s-%s-%s-%s-%s.csv", getResultDir(), filePrefix, toolName,
// workload, model, description);
FileUtils.write(FileUtils.getFile(targetDir, fileName), csv, Charset.defaultCharset());
} }
public String getResultDir() { // public String getResultDir() {
return workspaceDir + ResultHelper.getResultDirForId(benchmarkId); // return workspaceDir + ResultHelper.getResultDirForId(benchmarkId);
} // }
} }
...@@ -3,8 +3,15 @@ package hu.bme.mit.trainbenchmark.benchmark.result; ...@@ -3,8 +3,15 @@ package hu.bme.mit.trainbenchmark.benchmark.result;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.file.*; import java.nio.file.*;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfig;
import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfigBuilder;
import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
...@@ -12,6 +19,7 @@ public final class ResultHelper { ...@@ -12,6 +19,7 @@ public final class ResultHelper {
public static final String WORKSPACE_DIR = "../"; public static final String WORKSPACE_DIR = "../";
public static final String RESULT_DIR = "results/"; public static final String RESULT_DIR = "results/";
public static final String TOOLS_DIR = "tools/";
public static final String DIAGRAM_DIR = "diagrams/"; public static final String DIAGRAM_DIR = "diagrams/";
public static final String SCRIPT_PATH = WORKSPACE_DIR + "trainbenchmark-scripts/src/"; public static final String SCRIPT_PATH = WORKSPACE_DIR + "trainbenchmark-scripts/src/";
public static final String BENCHMARK_SCRIPT = "BenchmarkScript.groovy"; public static final String BENCHMARK_SCRIPT = "BenchmarkScript.groovy";
...@@ -94,4 +102,38 @@ public final class ResultHelper { ...@@ -94,4 +102,38 @@ public final class ResultHelper {
FileUtils.copyFile(srcFile, destFile); FileUtils.copyFile(srcFile, destFile);
} }
public static void createNewResultDirs(Collection<BenchmarkConfigBuilder<? extends BenchmarkConfig, ? extends BenchmarkConfigBuilder<?, ?>>> tools) {
}
/**
* Prepare a benchmark run, i.e., creates necessary directories. This operation is idempotent.
* @param config the benchmark configuration
* @param ec the execution configuration
*/
public static void prepare(BenchmarkConfig config, ExecutionConfig ec) {
// ensure directory for the tool
File toolDir = getToolDirectory(config.getProjectName());
toolDir.mkdir();
// ensure directory for the run inside toolDir
File runDir = getResultDirectory(toolDir, config.getConfigBase().getBenchmarkId());
runDir.mkdir();
}
public static String getNow() {
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
return dateFormat.format(new Date());
}
public static File getToolDirectory(String projectName) {
return FileUtils.getFile(WORKSPACE_DIR, RESULT_DIR, TOOLS_DIR, projectName);
}
public static File getResultDirectory(File toolDirectory, String benchmarkId) {
return FileUtils.getFile(toolDirectory, "run-" + benchmarkId);
}
public static File getResultDirectory(String projectName, String benchmarkId) {
return getResultDirectory(getToolDirectory(projectName), benchmarkId);
}
} }
...@@ -11,14 +11,38 @@ import org.apache.commons.exec.Executor; ...@@ -11,14 +11,38 @@ import org.apache.commons.exec.Executor;
import org.apache.commons.exec.PumpStreamHandler; import org.apache.commons.exec.PumpStreamHandler;
import com.google.common.base.Joiner; import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableMap;
import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfig; import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfig;
import hu.bme.mit.trainbenchmark.benchmark.memory.MemoryResult; import hu.bme.mit.trainbenchmark.benchmark.memory.MemoryResult;
import hu.bme.mit.trainbenchmark.config.ExecutionConfig; import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
import org.apache.commons.exec.*;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Map;
public class BenchmarkRunner { public class BenchmarkRunner {
public static int runPerformanceBenchmark(final BenchmarkConfig bc, final ExecutionConfig ec) public static String dictFormat(String format, Map<String, Object> values) {
StringBuilder convFormat = new StringBuilder(format);
ArrayList<Object> valueList = new ArrayList<>();
int currentPos = 1;
for (String key : values.keySet()) {
String formatKey = "%(" + key + ")",
formatPos = "%" + Integer.toString(currentPos) + "$1s";
int index = -1;
while ((index = convFormat.indexOf(formatKey, index)) != -1) {
convFormat.replace(index, index + formatKey.length(), formatPos);
index += formatPos.length();
}
valueList.add(values.get(key));
++currentPos;
}
return String.format(convFormat.toString(), valueList.toArray());
}
public static int runPerformanceBenchmark(final BenchmarkConfig bc, final ExecutionConfig ec, String jvmSetting)
throws IOException, InterruptedException { throws IOException, InterruptedException {
final Joiner joiner = Joiner.on(", "); final Joiner joiner = Joiner.on(", ");
System.out.println("Running benchmark."); System.out.println("Running benchmark.");
...@@ -38,9 +62,17 @@ public class BenchmarkRunner { ...@@ -38,9 +62,17 @@ public class BenchmarkRunner {
final String jarPath = String.format("../%s/build/libs/%s-1.0.0-SNAPSHOT-fat.jar %s", projectName, projectName, final String jarPath = String.format("../%s/build/libs/%s-1.0.0-SNAPSHOT-fat.jar %s", projectName, projectName,
configPath); configPath);
final String javaCommand = String.format("java -Xms%s -Xmx%s -server -Xverify:none -jar %s %s", ec.getXms(), ec.getXmx(), jarPath, configPath); final String javaCommand = dictFormat("java " + jvmSetting + " -jar %(jarPath) %(configPath)",
ImmutableMap.of("Xms", ec.getXms(), "Xmx", ec.getXmx(),
"jarPath", jarPath, "configPath", configPath));
// final String javaCommand = String.format("java -Xms%s -Xmx%s -server -Xverify:none -jar %s %s", ec.getXms(), ec.getXmx(), jarPath, configPath);
// final String javaCommand = String.format("java -Xms%s -Xmx%s -server -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:StartFlightRecording=dumponexit=true -Xverify:none -jar %s %s", ec.getXms(), ec.getXmx(), jarPath, configPath); // final String javaCommand = String.format("java -Xms%s -Xmx%s -server -XX:+UnlockCommercialFeatures -XX:+FlightRecorder -XX:StartFlightRecording=dumponexit=true -Xverify:none -jar %s %s", ec.getXms(), ec.getXmx(), jarPath, configPath);
final CommandLine cmdLine = CommandLine.parse(javaCommand); final CommandLine cmdLine = CommandLine.parse(javaCommand);
if (ec.getDryRun()) {
System.out.println("Would have executed: "+ cmdLine.toString());
return 0;
}
final long timeoutInSeconds = bc.getConfigBase().getTimeout(); final long timeoutInSeconds = bc.getConfigBase().getTimeout();
final long timeoutInMilliseconds = timeoutInSeconds * 1000; final long timeoutInMilliseconds = timeoutInSeconds * 1000;
...@@ -63,7 +95,7 @@ public class BenchmarkRunner { ...@@ -63,7 +95,7 @@ public class BenchmarkRunner {
} }
public static int runMemoryBenchmark(final BenchmarkConfig bc, final ExecutionConfig defaultEc, public static int runMemoryBenchmark(final BenchmarkConfig bc, final ExecutionConfig defaultEc,
final int numberOfSteps) throws IOException, InterruptedException { final int numberOfSteps, String jvmSetting) throws IOException, InterruptedException {
// e.g. initialMaxMemory = 12800, we save this (as a final variable), so // e.g. initialMaxMemory = 12800, we save this (as a final variable), so
// that we will not exceed it // that we will not exceed it
// //
...@@ -79,7 +111,7 @@ public class BenchmarkRunner { ...@@ -79,7 +111,7 @@ public class BenchmarkRunner {
memoryQuantum /= 2; memoryQuantum /= 2;
final ExecutionConfig ec = new ExecutionConfig(currentMaxMemory, currentMaxMemory); final ExecutionConfig ec = new ExecutionConfig(currentMaxMemory, currentMaxMemory);
if (runPerformanceBenchmark(bc, ec) == 0) { if (runPerformanceBenchmark(bc, ec, jvmSetting) == 0) {
System.out.println("Execution finished, testing with less memory."); System.out.println("Execution finished, testing with less memory.");
System.out.println(); System.out.println();
currentMaxMemory -= memoryQuantum; currentMaxMemory -= memoryQuantum;
......
...@@ -23,7 +23,7 @@ public abstract class TrainBenchmarkTest { ...@@ -23,7 +23,7 @@ public abstract class TrainBenchmarkTest {
@Rule @Rule
public ErrorCollector collector = new ErrorCollector(); public ErrorCollector collector = new ErrorCollector();
protected final int benchmarkId = 0; protected final String benchmarkId = "";
protected ExecutionConfig executionConfig = ExecutionConfig.defaultExecutionConfig(); protected ExecutionConfig executionConfig = ExecutionConfig.defaultExecutionConfig();
protected final long timeout = 120; protected final long timeout = 120;
protected final int runs = 1; protected final int runs = 1;
......
...@@ -11,7 +11,7 @@ import hu.bme.mit.trainbenchmark.config.ExecutionConfig; ...@@ -11,7 +11,7 @@ import hu.bme.mit.trainbenchmark.config.ExecutionConfig;
public abstract class QueryTest { public abstract class QueryTest {
protected final int benchmarkId = 0; protected final String benchmarkId = "";
protected ExecutionConfig executionConfig = ExecutionConfig.defaultExecutionConfig(); protected ExecutionConfig executionConfig = ExecutionConfig.defaultExecutionConfig();
protected final long timeout = 120; protected final long timeout = 120;
protected final int runs = 1; protected final int runs = 1;
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment