Use separate process for each solver in benchmark run.

- new option skipOnErrors (only used for full/scenario-benchmark) to skip remaining repetitions of one solver on error

jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/FullBenchmarkMain.java

@@ -15,9 +15,7 @@ import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Objects;
+import java.util.*;
 import java.util.stream.Collectors;
 
 /**
@@ -35,6 +33,8 @@ public class FullBenchmarkMain {
     private boolean useGradleConnection = false;
     private final String gradleCommand = System.getProperty("os.name")
         .toLowerCase().contains("windows") ? "./gradle.bat" : "./gradlew";
+    private final String gradleTaskName = "sandboxRun";
+    private final String gradleArgumentPrefix = "-PrunSettingsFile=";
 
     InvokeJavaVMBenchmark(BenchmarkSettings settings, int repetitions) {
       this.settings = settings;
@@ -48,7 +48,10 @@ public class FullBenchmarkMain {
 
     @Override
     public void run() {
-      for (int i = 0; i < this.repetitions; i++) {
+      // create a settings file for each solver
+      List<Path> pathList = new ArrayList<>();
+      for (String solverName : this.settings.solvers) {
+        settings.solvers = Collections.singletonList(solverName);
         Path tempPath;
         try {
           tempPath = Files.createTempFile("runSettings", ".json");
@@ -57,33 +60,41 @@ public class FullBenchmarkMain {
           logger.catching(e);
           throw new RuntimeException("Could not create temporary file for sandBoxed use. Exiting.");
         }
-        if (useGradleConnection) {
-          // invoke Gradle target directly
-          ProjectConnection connection = GradleConnector.newConnector()
-              .forProjectDirectory(new File("."))
-              .connect();
-          try {
-            connection.newBuild()
-                .forTasks("sandboxRun")
-                .withArguments("-PrunSettingsFile=" + tempPath.toAbsolutePath().toString())
-                .setStandardOutput(System.out)
-                .run();
-          } finally {
-            connection.close();
-          }
-        } else {
-          // start a new process calling gradle
-          File currentDir = getCurrentDir();
-          try {
-            ProcessBuilder pb = new ProcessBuilder(gradleCommand, "sandboxRun",
-                "-PrunSettingsFile=" + tempPath.toAbsolutePath().toString());
-            pb.redirectOutput(ProcessBuilder.Redirect.INHERIT);
-            pb.redirectError(ProcessBuilder.Redirect.INHERIT);
-            pb.directory(currentDir);
-            Process p = pb.start();
-            p.waitFor();
-          } catch (IOException | InterruptedException e) {
-            e.printStackTrace();
+        pathList.add(tempPath);
+      }
+      nextSolver: for (Path path : pathList) {
+        for (int i = 0; i < this.repetitions; i++) {
+          if (useGradleConnection) {
+            // invoke Gradle target directly
+            ProjectConnection connection = GradleConnector.newConnector()
+                .forProjectDirectory(new File("."))
+                .connect();
+            try {
+              connection.newBuild()
+                  .forTasks(gradleTaskName)
+                  .withArguments(gradleArgumentPrefix + path.toAbsolutePath().toString())
+                  .setStandardOutput(System.out)
+                  .run();
+            } finally {
+              connection.close();
+            }
+          } else {
+            // start a new process calling gradle
+            File currentDir = getCurrentDir();
+            try {
+              ProcessBuilder pb = new ProcessBuilder(gradleCommand, gradleTaskName,
+                  gradleArgumentPrefix + path.toAbsolutePath().toString());
+              pb.redirectOutput(ProcessBuilder.Redirect.INHERIT);
+              pb.redirectError(ProcessBuilder.Redirect.INHERIT);
+              pb.directory(currentDir);
+              Process p = pb.start();
+              p.waitFor();
+            } catch (IOException | InterruptedException e) {
+              e.printStackTrace();
+              if (settings.skipOnError) {
+                continue nextSolver;
+              }
+            }
           }
         }
       }
@@ -150,6 +161,7 @@ public class FullBenchmarkMain {
     result.logLevel = settings.logLevel;
     result.path = settings.path;
     result.solvers = settings.solvers;
+    result.skipOnError = settings.skipOnError;
     TestGeneratorSettings tgs = new TestGeneratorSettings();
     tgs.minTopLevelComponents = tgs.maxTopLevelComponents = 1;
     tgs.minAvgNumImplSubComponents = tgs.maxAvgNumImplSubComponents = 0;

jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/data/BenchmarkSettings.java

@@ -17,6 +17,7 @@ public class BenchmarkSettings {
   public String solutionFilePattern = null;
   public List<String> solvers = null;
   public String logLevel = null;
+  public Boolean skipOnError = true;
 
   public final TestGeneratorSettings basic = new TestGeneratorSettings();
 
@@ -26,6 +27,7 @@ public class BenchmarkSettings {
     this.resultFilePattern = nonNullOrDefault(other.resultFilePattern, this.resultFilePattern);
     this.solvers = nonNullOrDefault(other.solvers, this.solvers);
     this.logLevel = nonNullOrDefault(other.logLevel, this.logLevel);
+    this.skipOnError = nonNullOrDefault(other.skipOnError, this.skipOnError);
     updateBasic(other.basic);
   }
 

jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/data/ScenarioSettings.java

@@ -18,4 +18,5 @@ public class ScenarioSettings {
   public List<ScenarioData> scenarios;
   public boolean sandBoxed;
   public boolean useGradleConnection = false;
+  public boolean skipOnError = true;
 }