Commit fd919675 authored by René Schöne's avatar René Schöne
Browse files

Use separate process for each solver in benchmark run.

- new option skipOnErrors (only used for full/scenario-benchmark) to skip remaining repetitions of one solver on error
parent 3b037fb2
......@@ -15,9 +15,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.*;
import java.util.stream.Collectors;
/**
......@@ -35,6 +33,8 @@ public class FullBenchmarkMain {
private boolean useGradleConnection = false;
private final String gradleCommand = System.getProperty("os.name")
.toLowerCase().contains("windows") ? "./gradle.bat" : "./gradlew";
private final String gradleTaskName = "sandboxRun";
private final String gradleArgumentPrefix = "-PrunSettingsFile=";
InvokeJavaVMBenchmark(BenchmarkSettings settings, int repetitions) {
this.settings = settings;
......@@ -48,7 +48,10 @@ public class FullBenchmarkMain {
@Override
public void run() {
for (int i = 0; i < this.repetitions; i++) {
// create a settings file for each solver
List<Path> pathList = new ArrayList<>();
for (String solverName : this.settings.solvers) {
settings.solvers = Collections.singletonList(solverName);
Path tempPath;
try {
tempPath = Files.createTempFile("runSettings", ".json");
......@@ -57,33 +60,41 @@ public class FullBenchmarkMain {
logger.catching(e);
throw new RuntimeException("Could not create temporary file for sandBoxed use. Exiting.");
}
if (useGradleConnection) {
// invoke Gradle target directly
ProjectConnection connection = GradleConnector.newConnector()
.forProjectDirectory(new File("."))
.connect();
try {
connection.newBuild()
.forTasks("sandboxRun")
.withArguments("-PrunSettingsFile=" + tempPath.toAbsolutePath().toString())
.setStandardOutput(System.out)
.run();
} finally {
connection.close();
}
} else {
// start a new process calling gradle
File currentDir = getCurrentDir();
try {
ProcessBuilder pb = new ProcessBuilder(gradleCommand, "sandboxRun",
"-PrunSettingsFile=" + tempPath.toAbsolutePath().toString());
pb.redirectOutput(ProcessBuilder.Redirect.INHERIT);
pb.redirectError(ProcessBuilder.Redirect.INHERIT);
pb.directory(currentDir);
Process p = pb.start();
p.waitFor();
} catch (IOException | InterruptedException e) {
e.printStackTrace();
pathList.add(tempPath);
}
nextSolver: for (Path path : pathList) {
for (int i = 0; i < this.repetitions; i++) {
if (useGradleConnection) {
// invoke Gradle target directly
ProjectConnection connection = GradleConnector.newConnector()
.forProjectDirectory(new File("."))
.connect();
try {
connection.newBuild()
.forTasks(gradleTaskName)
.withArguments(gradleArgumentPrefix + path.toAbsolutePath().toString())
.setStandardOutput(System.out)
.run();
} finally {
connection.close();
}
} else {
// start a new process calling gradle
File currentDir = getCurrentDir();
try {
ProcessBuilder pb = new ProcessBuilder(gradleCommand, gradleTaskName,
gradleArgumentPrefix + path.toAbsolutePath().toString());
pb.redirectOutput(ProcessBuilder.Redirect.INHERIT);
pb.redirectError(ProcessBuilder.Redirect.INHERIT);
pb.directory(currentDir);
Process p = pb.start();
p.waitFor();
} catch (IOException | InterruptedException e) {
e.printStackTrace();
if (settings.skipOnError) {
continue nextSolver;
}
}
}
}
}
......@@ -150,6 +161,7 @@ public class FullBenchmarkMain {
result.logLevel = settings.logLevel;
result.path = settings.path;
result.solvers = settings.solvers;
result.skipOnError = settings.skipOnError;
TestGeneratorSettings tgs = new TestGeneratorSettings();
tgs.minTopLevelComponents = tgs.maxTopLevelComponents = 1;
tgs.minAvgNumImplSubComponents = tgs.maxAvgNumImplSubComponents = 0;
......
......@@ -17,6 +17,7 @@ public class BenchmarkSettings {
public String solutionFilePattern = null;
public List<String> solvers = null;
public String logLevel = null;
public Boolean skipOnError = true;
public final TestGeneratorSettings basic = new TestGeneratorSettings();
......@@ -26,6 +27,7 @@ public class BenchmarkSettings {
this.resultFilePattern = nonNullOrDefault(other.resultFilePattern, this.resultFilePattern);
this.solvers = nonNullOrDefault(other.solvers, this.solvers);
this.logLevel = nonNullOrDefault(other.logLevel, this.logLevel);
this.skipOnError = nonNullOrDefault(other.skipOnError, this.skipOnError);
updateBasic(other.basic);
}
......
......@@ -18,4 +18,5 @@ public class ScenarioSettings {
public List<ScenarioData> scenarios;
public boolean sandBoxed;
public boolean useGradleConnection = false;
public boolean skipOnError = true;
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment