Use separate process for each solver in benchmark run.

- new option skipOnErrors (only used for full/scenario-benchmark) to skip remaining repetitions of one solver on error

jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/FullBenchmarkMain.java

@@ -15,9 +15,7 @@ import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Objects;
+import java.util.*;
 import java.util.stream.Collectors;
 
 /**
@@ -35,6 +33,8 @@ public class FullBenchmarkMain {
     private boolean useGradleConnection = false;
     private final String gradleCommand = System.getProperty("os.name")
         .toLowerCase().contains("windows") ? "./gradle.bat" : "./gradlew";
+    private final String gradleTaskName = "sandboxRun";
+    private final String gradleArgumentPrefix = "-PrunSettingsFile=";
 
     InvokeJavaVMBenchmark(BenchmarkSettings settings, int repetitions) {
       this.settings = settings;
@@ -48,7 +48,10 @@ public class FullBenchmarkMain {
 
     @Override
     public void run() {
-      for (int i = 0; i < this.repetitions; i++) {
+      // create a settings file for each solver
+      List<Path> pathList = new ArrayList<>();
+      for (String solverName : this.settings.solvers) {
+        settings.solvers = Collections.singletonList(solverName);
         Path tempPath;
         try {
           tempPath = Files.createTempFile("runSettings", ".json");
@@ -57,6 +60,10 @@ public class FullBenchmarkMain {
           logger.catching(e);
           throw new RuntimeException("Could not create temporary file for sandBoxed use. Exiting.");
         }
+        pathList.add(tempPath);
+      }
+      nextSolver: for (Path path : pathList) {
+        for (int i = 0; i < this.repetitions; i++) {
           if (useGradleConnection) {
             // invoke Gradle target directly
             ProjectConnection connection = GradleConnector.newConnector()
@@ -64,8 +71,8 @@ public class FullBenchmarkMain {
                 .connect();
             try {
               connection.newBuild()
-                .forTasks("sandboxRun")
-                .withArguments("-PrunSettingsFile=" + tempPath.toAbsolutePath().toString())
+                  .forTasks(gradleTaskName)
+                  .withArguments(gradleArgumentPrefix + path.toAbsolutePath().toString())
                   .setStandardOutput(System.out)
                   .run();
             } finally {
@@ -75,8 +82,8 @@ public class FullBenchmarkMain {
             // start a new process calling gradle
             File currentDir = getCurrentDir();
             try {
-            ProcessBuilder pb = new ProcessBuilder(gradleCommand, "sandboxRun",
-                "-PrunSettingsFile=" + tempPath.toAbsolutePath().toString());
+              ProcessBuilder pb = new ProcessBuilder(gradleCommand, gradleTaskName,
+                  gradleArgumentPrefix + path.toAbsolutePath().toString());
               pb.redirectOutput(ProcessBuilder.Redirect.INHERIT);
               pb.redirectError(ProcessBuilder.Redirect.INHERIT);
               pb.directory(currentDir);
@@ -84,6 +91,10 @@ public class FullBenchmarkMain {
               p.waitFor();
             } catch (IOException | InterruptedException e) {
               e.printStackTrace();
+              if (settings.skipOnError) {
+                continue nextSolver;
+              }
+            }
           }
         }
       }
@@ -150,6 +161,7 @@ public class FullBenchmarkMain {
     result.logLevel = settings.logLevel;
     result.path = settings.path;
     result.solvers = settings.solvers;
+    result.skipOnError = settings.skipOnError;
     TestGeneratorSettings tgs = new TestGeneratorSettings();
     tgs.minTopLevelComponents = tgs.maxTopLevelComponents = 1;
     tgs.minAvgNumImplSubComponents = tgs.maxAvgNumImplSubComponents = 0;

jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/data/BenchmarkSettings.java

@@ -17,6 +17,7 @@ public class BenchmarkSettings {
   public String solutionFilePattern = null;
   public List<String> solvers = null;
   public String logLevel = null;
+  public Boolean skipOnError = true;
 
   public final TestGeneratorSettings basic = new TestGeneratorSettings();
 
@@ -26,6 +27,7 @@ public class BenchmarkSettings {
     this.resultFilePattern = nonNullOrDefault(other.resultFilePattern, this.resultFilePattern);
     this.solvers = nonNullOrDefault(other.solvers, this.solvers);
     this.logLevel = nonNullOrDefault(other.logLevel, this.logLevel);
+    this.skipOnError = nonNullOrDefault(other.skipOnError, this.skipOnError);
     updateBasic(other.basic);
   }
 

jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/data/ScenarioSettings.java

@@ -18,4 +18,5 @@ public class ScenarioSettings {
   public List<ScenarioData> scenarios;
   public boolean sandBoxed;
   public boolean useGradleConnection = false;
+  public boolean skipOnError = true;
 }