From b0a7d1c4dd2fb32e71702dff0ddeb9290032409c Mon Sep 17 00:00:00 2001
From: rschoene <rene.schoene@tu-dresden.de>
Date: Tue, 12 Jun 2018 11:01:46 +0200
Subject: [PATCH] Enable sand-boxed benchmark runs.

---
 jastadd-mquat-benchmark/build.gradle          | 14 ++++++
 .../st/mquat/benchmark/FullBenchmarkMain.java | 50 ++++++++++++++++++-
 .../st/mquat/benchmark/SandboxRunMain.java    | 44 ++++++++++++++++
 .../inf/st/mquat/benchmark/Utils.java         |  9 +++-
 .../benchmark/data/ScenarioSettings.java      |  1 +
 .../src/main/resources/scenarios.json         |  1 +
 6 files changed, 117 insertions(+), 2 deletions(-)
 create mode 100644 jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/SandboxRunMain.java

diff --git a/jastadd-mquat-benchmark/build.gradle b/jastadd-mquat-benchmark/build.gradle
index eaafbc0..3ecf950 100644
--- a/jastadd-mquat-benchmark/build.gradle
+++ b/jastadd-mquat-benchmark/build.gradle
@@ -2,9 +2,11 @@
 apply plugin: 'java'
 
 sourceCompatibility = 1.8
+def toolingApiVersion = 3.3
 
 repositories {
     mavenCentral()
+    maven { url 'https://repo.gradle.org/gradle/libs-releases' }
 }
 
 dependencies {
@@ -14,6 +16,8 @@ dependencies {
     compile project(':jastadd-mquat-solver-simple')
     compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.8.8.1'
     compile group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.10.0'
+    compile "org.gradle:gradle-tooling-api:${toolingApiVersion}"
+    runtime 'org.slf4j:slf4j-log4j12:1.7.10'
     testCompile group: 'junit', name: 'junit', version: '4.12'
 }
 
@@ -36,3 +40,13 @@ task benchmarkFull(type: JavaExec, dependsOn: assemble) {
     }
 //    jvmArgs("-XX:+UnlockCommercialFeatures", "-XX:+FlightRecorder", "-XX:StartFlightRecording=settings=profile", "-XX:FlightRecorderOptions=defaultrecording=true,dumponexit=true,dumponexitpath=results/fr", "-Xverify:none")
 }
+
+task sandboxRun(type: JavaExec) {
+    classpath = sourceSets.test.runtimeClasspath
+    main = 'de.tudresden.inf.st.mquat.benchmark.SandboxRunMain'
+    systemProperty "java.library.path", project.glpkPath
+//    jvmArgs("-Xmx=4096m")
+    if (project.hasProperty("runSettingsFile")) {
+        args(runSettingsFile)
+    }
+}
diff --git a/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/FullBenchmarkMain.java b/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/FullBenchmarkMain.java
index 403d8b6..706ff95 100644
--- a/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/FullBenchmarkMain.java
+++ b/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/FullBenchmarkMain.java
@@ -8,8 +8,13 @@ import de.tudresden.inf.st.mquat.benchmark.data.ScenarioSettings;
 import de.tudresden.inf.st.mquat.solving.BenchmarkableSolver;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.gradle.tooling.GradleConnector;
+import org.gradle.tooling.ProjectConnection;
 
+import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Objects;
@@ -24,6 +29,43 @@ public class FullBenchmarkMain {
 
   private static Logger logger = LogManager.getLogger(FullBenchmarkMain.class);
 
+  static class InvokeJavaVMBenchmark extends Benchmark {
+    BenchmarkSettings settings;
+    int repetitions;
+
+    InvokeJavaVMBenchmark(BenchmarkSettings settings, int repetitions) {
+      this.settings = settings;
+      this.repetitions = repetitions;
+    }
+
+    @Override
+    public void run() {
+      for (int i = 0; i < this.repetitions; i++) {
+        Path tempPath;
+        try {
+          tempPath = Files.createTempFile("runSettings", ".json");
+          Utils.writeToResource(Utils.getMapper(), tempPath, settings);
+        } catch (IOException e) {
+          logger.catching(e);
+          throw new RuntimeException("Could not create temporary file for sandBoxed use. Exiting.");
+        }
+        // invoke Gradle target
+        ProjectConnection connection = GradleConnector.newConnector()
+            .forProjectDirectory(new File("."))
+            .connect();
+        try {
+          connection.newBuild()
+              .forTasks("sandboxRun")
+              .withArguments("-PrunSettingsFile=" + tempPath.toAbsolutePath().toString())
+              .setStandardOutput(System.out)
+              .run();
+        } finally {
+          connection.close();
+        }
+      }
+    }
+  }
+
   public static void main(String[] args) {
     List<Benchmark> benchmarks = createFromConfig(args);
     if (benchmarks == null || benchmarks.isEmpty()) {
@@ -53,7 +95,7 @@ public class FullBenchmarkMain {
         .map(SolverFactory::getSolverByName).collect(Collectors.toList());
     return settings.scenarios.stream()
         .filter(data -> takeAll || allowedIds.contains(data.getId()) || allowedNames.contains(data.name))
-        .map(data -> new ScenarioBenchmark(from(settings, data), solvers, settings.repetitions))
+        .map(data -> create(from(settings, data), solvers, settings.repetitions, settings.sandBoxed))
         .collect(Collectors.toList());
   }
 
@@ -101,4 +143,10 @@ public class FullBenchmarkMain {
     return result;
   }
 
+  private static Benchmark create(BenchmarkSettings settings, List<BenchmarkableSolver> solvers, int repetitions, boolean sandBoxed) {
+    return sandBoxed ?
+        new InvokeJavaVMBenchmark(settings, repetitions) :
+        new ScenarioBenchmark(settings, solvers, repetitions);
+  }
+
 }
diff --git a/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/SandboxRunMain.java b/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/SandboxRunMain.java
new file mode 100644
index 0000000..75deccd
--- /dev/null
+++ b/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/SandboxRunMain.java
@@ -0,0 +1,44 @@
+package de.tudresden.inf.st.mquat.benchmark;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import de.tudresden.inf.st.mquat.benchmark.data.BenchmarkSettings;
+import de.tudresden.inf.st.mquat.solving.BenchmarkableSolver;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Class to be executed to run a benchmark in a separate JavaVM.
+ *
+ * @author rschoene - Initial contribution
+ */
+public class SandboxRunMain {
+
+  private static final Logger logger = LogManager.getLogger(SandboxRunMain.class);
+
+  public static void main(String[] args) {
+    // expect runSettingsFile as first argument
+    String runSettingsFilename = args[0];
+    // construct a ScenarioBenchmark from this setting
+    // read settings first
+    ObjectMapper mapper = Utils.getMapper();
+    BenchmarkSettings settings;
+    try {
+      settings = mapper.readValue(new File(runSettingsFilename), BenchmarkSettings.class);
+    } catch (IOException e) {
+      logger.catching(e);
+      throw new RuntimeException("Could not read settings! Exiting.", e);
+    }
+    // create solvers
+    final List<BenchmarkableSolver> solvers = settings.solvers.stream()
+        .map(SolverFactory::getSolverByName).collect(Collectors.toList());
+    // repetitions will always be controlled outside of this sand-boxed environment
+    ScenarioBenchmark benchmark = new ScenarioBenchmark(settings, solvers, 1);
+    benchmark.run();
+  }
+
+}
diff --git a/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/Utils.java b/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/Utils.java
index 791e011..ef4fa8c 100644
--- a/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/Utils.java
+++ b/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/Utils.java
@@ -6,6 +6,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.File;
 import java.io.IOException;
 import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
 
 public class Utils {
 
@@ -16,7 +18,7 @@ public class Utils {
   }
 
   private static File readFromResource(String filename) throws IOException {
-    URL basicSettingsURL = CustomBenchmarkMain.class.getClassLoader().getResource(filename);
+    URL basicSettingsURL = Utils.class.getClassLoader().getResource(filename);
     if (basicSettingsURL == null) {
       System.err.println();
       throw new IOException("Could not access " + filename + ". Exiting.");
@@ -37,6 +39,11 @@ public class Utils {
     return result;
   }
 
+  static void writeToResource(ObjectMapper mapper, Path path, Object object) throws IOException {
+    File resultFile = path.toFile();
+    mapper.writeValue(resultFile, object);
+  }
+
   public static <T> T nonNullOrDefault(T newValue, T defaultValue) {
     return newValue != null ? newValue : defaultValue;
   }
diff --git a/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/data/ScenarioSettings.java b/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/data/ScenarioSettings.java
index fe78554..9614f3f 100644
--- a/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/data/ScenarioSettings.java
+++ b/jastadd-mquat-benchmark/src/main/java/de/tudresden/inf/st/mquat/benchmark/data/ScenarioSettings.java
@@ -16,4 +16,5 @@ public class ScenarioSettings {
   public int seed;
   public int repetitions = 1;
   public List<ScenarioData> scenarios;
+  public boolean sandBoxed;
 }
diff --git a/jastadd-mquat-benchmark/src/main/resources/scenarios.json b/jastadd-mquat-benchmark/src/main/resources/scenarios.json
index 42a8d7b..7a027be 100644
--- a/jastadd-mquat-benchmark/src/main/resources/scenarios.json
+++ b/jastadd-mquat-benchmark/src/main/resources/scenarios.json
@@ -9,6 +9,7 @@
   "timeoutValue": 15,
   "timeoutUnit": "MINUTES",
   "seed": 0,
+  "sandBoxed": true,
 //  "repetitions": 10,
   "scenarios": [
     { "id": 0, "name": "trivial", "variants": 1, "requests": 1, "depth": 1, "resources": 1 },
-- 
GitLab