From fd919675fa74067d531d4cccf44bfc6ecb3649ed Mon Sep 17 00:00:00 2001 From: rschoene <rene.schoene@tu-dresden.de> Date: Thu, 14 Jun 2018 14:28:59 +0200 Subject: [PATCH] Use separate process for each solver in benchmark run. - new option skipOnErrors (only used for full/scenario-benchmark) to skip remaining repetitions of one solver on error --- .../st/mquat/benchmark/FullBenchmarkMain.java | 74 +++++++++++-------- .../benchmark/data/BenchmarkSettings.java | 2 + .../benchmark/data/ScenarioSettings.java | 1 + 3 files changed, 46 insertions(+), 31 deletions(-) diff --git a/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/FullBenchmarkMain.java b/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/FullBenchmarkMain.java index 4fcbbe0..40d0bc7 100644 --- a/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/FullBenchmarkMain.java +++ b/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/FullBenchmarkMain.java @@ -15,9 +15,7 @@ import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; +import java.util.*; import java.util.stream.Collectors; /** @@ -35,6 +33,8 @@ public class FullBenchmarkMain { private boolean useGradleConnection = false; private final String gradleCommand = System.getProperty("os.name") .toLowerCase().contains("windows") ? "./gradle.bat" : "./gradlew"; + private final String gradleTaskName = "sandboxRun"; + private final String gradleArgumentPrefix = "-PrunSettingsFile="; InvokeJavaVMBenchmark(BenchmarkSettings settings, int repetitions) { this.settings = settings; @@ -48,7 +48,10 @@ public class FullBenchmarkMain { @Override public void run() { - for (int i = 0; i < this.repetitions; i++) { + // create a settings file for each solver + List<Path> pathList = new ArrayList<>(); + for (String solverName : this.settings.solvers) { + settings.solvers = Collections.singletonList(solverName); Path tempPath; try { tempPath = Files.createTempFile("runSettings", ".json"); @@ -57,33 +60,41 @@ public class FullBenchmarkMain { logger.catching(e); throw new RuntimeException("Could not create temporary file for sandBoxed use. Exiting."); } - if (useGradleConnection) { - // invoke Gradle target directly - ProjectConnection connection = GradleConnector.newConnector() - .forProjectDirectory(new File(".")) - .connect(); - try { - connection.newBuild() - .forTasks("sandboxRun") - .withArguments("-PrunSettingsFile=" + tempPath.toAbsolutePath().toString()) - .setStandardOutput(System.out) - .run(); - } finally { - connection.close(); - } - } else { - // start a new process calling gradle - File currentDir = getCurrentDir(); - try { - ProcessBuilder pb = new ProcessBuilder(gradleCommand, "sandboxRun", - "-PrunSettingsFile=" + tempPath.toAbsolutePath().toString()); - pb.redirectOutput(ProcessBuilder.Redirect.INHERIT); - pb.redirectError(ProcessBuilder.Redirect.INHERIT); - pb.directory(currentDir); - Process p = pb.start(); - p.waitFor(); - } catch (IOException | InterruptedException e) { - e.printStackTrace(); + pathList.add(tempPath); + } + nextSolver: for (Path path : pathList) { + for (int i = 0; i < this.repetitions; i++) { + if (useGradleConnection) { + // invoke Gradle target directly + ProjectConnection connection = GradleConnector.newConnector() + .forProjectDirectory(new File(".")) + .connect(); + try { + connection.newBuild() + .forTasks(gradleTaskName) + .withArguments(gradleArgumentPrefix + path.toAbsolutePath().toString()) + .setStandardOutput(System.out) + .run(); + } finally { + connection.close(); + } + } else { + // start a new process calling gradle + File currentDir = getCurrentDir(); + try { + ProcessBuilder pb = new ProcessBuilder(gradleCommand, gradleTaskName, + gradleArgumentPrefix + path.toAbsolutePath().toString()); + pb.redirectOutput(ProcessBuilder.Redirect.INHERIT); + pb.redirectError(ProcessBuilder.Redirect.INHERIT); + pb.directory(currentDir); + Process p = pb.start(); + p.waitFor(); + } catch (IOException | InterruptedException e) { + e.printStackTrace(); + if (settings.skipOnError) { + continue nextSolver; + } + } } } } @@ -150,6 +161,7 @@ public class FullBenchmarkMain { result.logLevel = settings.logLevel; result.path = settings.path; result.solvers = settings.solvers; + result.skipOnError = settings.skipOnError; TestGeneratorSettings tgs = new TestGeneratorSettings(); tgs.minTopLevelComponents = tgs.maxTopLevelComponents = 1; tgs.minAvgNumImplSubComponents = tgs.maxAvgNumImplSubComponents = 0; diff --git a/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/data/BenchmarkSettings.java b/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/data/BenchmarkSettings.java index 998df5e..e4e5797 100644 --- a/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/data/BenchmarkSettings.java +++ b/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/data/BenchmarkSettings.java @@ -17,6 +17,7 @@ public class BenchmarkSettings { public String solutionFilePattern = null; public List<String> solvers = null; public String logLevel = null; + public Boolean skipOnError = true; public final TestGeneratorSettings basic = new TestGeneratorSettings(); @@ -26,6 +27,7 @@ public class BenchmarkSettings { this.resultFilePattern = nonNullOrDefault(other.resultFilePattern, this.resultFilePattern); this.solvers = nonNullOrDefault(other.solvers, this.solvers); this.logLevel = nonNullOrDefault(other.logLevel, this.logLevel); + this.skipOnError = nonNullOrDefault(other.skipOnError, this.skipOnError); updateBasic(other.basic); } diff --git a/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/data/ScenarioSettings.java b/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/data/ScenarioSettings.java index 3d2fa73..b8b34bf 100644 --- a/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/data/ScenarioSettings.java +++ b/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/data/ScenarioSettings.java @@ -18,4 +18,5 @@ public class ScenarioSettings { public List<ScenarioData> scenarios; public boolean sandBoxed; public boolean useGradleConnection = false; + public boolean skipOnError = true; } -- GitLab