Select Git revision
BenchmarkScript.groovy

Johannes Mey authored
BenchmarkScript.groovy 6.34 KiB
import hu.bme.mit.trainbenchmark.config.ExecutionConfig
import hu.bme.mit.trainbenchmark.constants.RailwayOperation
import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfigBaseBuilder
import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfigBuilder
import hu.bme.mit.trainbenchmark.benchmark.config.ModelSetConfig
import hu.bme.mit.trainbenchmark.benchmark.config.TransformationChangeSetStrategy
import hu.bme.mit.trainbenchmark.benchmark.result.ResultHelper
import hu.bme.mit.trainbenchmark.benchmark.runcomponents.BenchmarkReporter
import hu.bme.mit.trainbenchmark.benchmark.runcomponents.BenchmarkRunner
// tools
import de.tudresden.inf.st.train.jastadd.config.JastaddNameLookupBenchmarkConfigBuilder
import de.tudresden.inf.st.train.jastadd.config.JastaddNameLookupIncrementalBenchmarkConfigBuilder
import de.tudresden.inf.st.train.jastadd.config.JastaddOptimizedBenchmarkConfigBuilder
import de.tudresden.inf.st.train.jastadd.config.JastaddOptimizedIncrementalBenchmarkConfigBuilder
import de.tudresden.inf.st.train.jastadd.config.JastaddSpecializedBenchmarkConfigBuilder
import de.tudresden.inf.st.train.jastadd.config.JastaddSpecializedIncrementalBenchmarkConfigBuilder
import hu.bme.mit.trainbenchmark.benchmark.tinkergraph.config.TinkerGraphBenchmarkConfigBuilder
import hu.bme.mit.trainbenchmark.benchmark.viatra.config.ViatraBackend
import hu.bme.mit.trainbenchmark.benchmark.viatra.config.ViatraBenchmarkConfigBuilder
println('Please remember to stop all other Java processes.')
println()
println('If in doubt, check with this command:')
println('$ ps auxw | grep jav[a]')
println()
println('If there are other Java processes, use:')
println('$ killall -9 java')
def benchmarkId = ResultHelper.createNewResultDir()
ResultHelper.saveConfiguration(benchmarkId, "IndividualBenchmarkInjectScript.groovy")
def ec = new ExecutionConfig(8000, 32000)
def minSize = 1
def maxSize = 512
def timeout = 900
def runs = 10
println()
println("############################################################")
println('Benchmark parameters:')
println("- execution config: ${ec}")
println("- range: minSize=${minSize}, maxSize=${maxSize}")
println("- timeout: ${timeout}")
println("- runs: ${runs}")
println("############################################################")
println()
// Set the reportUrl if you would like to receive a Slack notification when the benchmark finished.
//reportUrl = "https://hooks.slack.com/services/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"
def tools = [
new JastaddNameLookupBenchmarkConfigBuilder(),
new JastaddNameLookupIncrementalBenchmarkConfigBuilder(),
new JastaddOptimizedBenchmarkConfigBuilder(),
new JastaddOptimizedIncrementalBenchmarkConfigBuilder(),
new JastaddSpecializedBenchmarkConfigBuilder(),
new JastaddSpecializedIncrementalBenchmarkConfigBuilder(),
new TinkerGraphBenchmarkConfigBuilder(),
new ViatraBenchmarkConfigBuilder().setBackend(ViatraBackend.INCREMENTAL),
]
def workloads = [
ConnectedSegments: [ modelVariant: "inject", operations: [RailwayOperation.CONNECTEDSEGMENTS, RailwayOperation.CONNECTEDSEGMENTS_INJECT], strategy: TransformationChangeSetStrategy.FIXED, constant: 10, queryTransformationCount: 12],
PosLength: [ modelVariant: "inject", operations: [RailwayOperation.POSLENGTH , RailwayOperation.POSLENGTH_INJECT ], strategy: TransformationChangeSetStrategy.FIXED, constant: 10, queryTransformationCount: 12],
RouteSensor: [ modelVariant: "inject", operations: [RailwayOperation.ROUTESENSOR , RailwayOperation.ROUTESENSOR_INJECT ], strategy: TransformationChangeSetStrategy.FIXED, constant: 10, queryTransformationCount: 12],
SemaphoreNeighbor: [ modelVariant: "inject", operations: [RailwayOperation.SEMAPHORENEIGHBOR, RailwayOperation.SEMAPHORENEIGHBOR_INJECT], strategy: TransformationChangeSetStrategy.FIXED, constant: 10, queryTransformationCount: 12],
SwitchMonitored: [ modelVariant: "inject", operations: [RailwayOperation.SWITCHMONITORED , RailwayOperation.SWITCHMONITORED_INJECT ], strategy: TransformationChangeSetStrategy.FIXED, constant: 10, queryTransformationCount: 12],
SwitchSet: [ modelVariant: "inject", operations: [RailwayOperation.SWITCHSET , RailwayOperation.SWITCHSET_INJECT ], strategy: TransformationChangeSetStrategy.FIXED, constant: 10, queryTransformationCount: 12],
]
def runBenchmarkSeries(BenchmarkConfigBaseBuilder configBaseBuilder, BenchmarkConfigBuilder configBuilder,
ExecutionConfig ec, ModelSetConfig modelSetConfig) {
try {
for (def size = modelSetConfig.minSize; size <= modelSetConfig.maxSize; size *= 2) {
def modelFilename = "railway-${modelSetConfig.modelVariant}-${size}"
println("------------------------------------------------------------")
println("Model: $modelFilename")
println("------------------------------------------------------------")
configBaseBuilder.setModelFilename(modelFilename)
def configBase = configBaseBuilder.createConfigBase()
def config = configBuilder.setConfigBase(configBase).createConfig()
def exitValue = BenchmarkRunner.runPerformanceBenchmark(config, ec)
if (exitValue != 0) {
println "Timeout or error occured, skipping models for larger sizes. Error code: ${exitValue}"
break
}
}
} catch (all) {
println "Exception occured during execution."
}
}
workloads.each { workload ->
def workloadName = workload.key
def workloadConfiguration = workload.value
def modelVariant = workloadConfiguration["modelVariant"]
def operations = workloadConfiguration["operations"]
def strategy = workloadConfiguration["strategy"]
def constant = workloadConfiguration["constant"]
def queryTransformationCount = workloadConfiguration["queryTransformationCount"]
println("============================================================")
println("Workload: $workloadName")
println("============================================================")
def modelSetConfig = new ModelSetConfig(modelVariant, minSize, maxSize)
def bcbb = new BenchmarkConfigBaseBuilder()
.setBenchmarkId(benchmarkId).setTimeout(timeout).setRuns(runs)
.setOperations(operations).setWorkload(workloadName)
.setQueryTransformationCount(queryTransformationCount).setTransformationConstant(constant)
.setTransformationChangeSetStrategy(strategy)
tools.each{ bcb -> runBenchmarkSeries(bcbb, bcb, ec, modelSetConfig) }
}
if (binding.variables.get("reportUrl")) {
BenchmarkReporter.reportReady(reportUrl)
}