Skip to content
Snippets Groups Projects
Select Git revision
  • 1284b61466cbe525bbbcc00e3c28a5e736165e71
  • master default protected
  • restructure-2021
  • ae-submission
  • journal-diagrams
  • artifact-evaluation
  • artifact-evaluation-poster
  • ci
8 results

BenchmarkScript.groovy

Blame
  • BenchmarkScript.groovy 6.34 KiB
    import hu.bme.mit.trainbenchmark.config.ExecutionConfig
    import hu.bme.mit.trainbenchmark.constants.RailwayOperation
    import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfigBaseBuilder
    import hu.bme.mit.trainbenchmark.benchmark.config.BenchmarkConfigBuilder
    import hu.bme.mit.trainbenchmark.benchmark.config.ModelSetConfig
    import hu.bme.mit.trainbenchmark.benchmark.config.TransformationChangeSetStrategy
    import hu.bme.mit.trainbenchmark.benchmark.result.ResultHelper
    import hu.bme.mit.trainbenchmark.benchmark.runcomponents.BenchmarkReporter
    import hu.bme.mit.trainbenchmark.benchmark.runcomponents.BenchmarkRunner
    
    // tools
    import de.tudresden.inf.st.train.jastadd.config.JastaddNameLookupBenchmarkConfigBuilder
    import de.tudresden.inf.st.train.jastadd.config.JastaddNameLookupIncrementalBenchmarkConfigBuilder
    import de.tudresden.inf.st.train.jastadd.config.JastaddOptimizedBenchmarkConfigBuilder
    import de.tudresden.inf.st.train.jastadd.config.JastaddOptimizedIncrementalBenchmarkConfigBuilder
    import de.tudresden.inf.st.train.jastadd.config.JastaddSpecializedBenchmarkConfigBuilder
    import de.tudresden.inf.st.train.jastadd.config.JastaddSpecializedIncrementalBenchmarkConfigBuilder
    import hu.bme.mit.trainbenchmark.benchmark.tinkergraph.config.TinkerGraphBenchmarkConfigBuilder
    import hu.bme.mit.trainbenchmark.benchmark.viatra.config.ViatraBackend
    import hu.bme.mit.trainbenchmark.benchmark.viatra.config.ViatraBenchmarkConfigBuilder
    
    println('Please remember to stop all other Java processes.')
    println()
    println('If in doubt, check with this command:')
    println('$ ps auxw | grep jav[a]')
    println()
    println('If there are other Java processes, use:')
    println('$ killall -9 java')
    
    def benchmarkId = ResultHelper.createNewResultDir()
    ResultHelper.saveConfiguration(benchmarkId, "IndividualBenchmarkInjectScript.groovy")
    def ec = new ExecutionConfig(8000, 32000)
    
    def minSize = 1
    def maxSize = 512
    def timeout = 900
    def runs = 10
    
    println()
    println("############################################################")
    println('Benchmark parameters:')
    println("- execution config: ${ec}")
    println("- range: minSize=${minSize}, maxSize=${maxSize}")
    println("- timeout: ${timeout}")
    println("- runs: ${runs}")
    println("############################################################")
    println()
    
    // Set the reportUrl if you would like to receive a Slack notification when the benchmark finished.
    //reportUrl = "https://hooks.slack.com/services/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"
    
    def tools = [
    		new JastaddNameLookupBenchmarkConfigBuilder(),
    		new JastaddNameLookupIncrementalBenchmarkConfigBuilder(),
    		new JastaddOptimizedBenchmarkConfigBuilder(),
    		new JastaddOptimizedIncrementalBenchmarkConfigBuilder(),
    		new JastaddSpecializedBenchmarkConfigBuilder(),
    		new JastaddSpecializedIncrementalBenchmarkConfigBuilder(),
    		new TinkerGraphBenchmarkConfigBuilder(),
    		new ViatraBenchmarkConfigBuilder().setBackend(ViatraBackend.INCREMENTAL),
    ]
    
    def workloads = [
    	ConnectedSegments: [ modelVariant: "inject", operations: [RailwayOperation.CONNECTEDSEGMENTS, RailwayOperation.CONNECTEDSEGMENTS_INJECT], strategy: TransformationChangeSetStrategy.FIXED, constant: 10, queryTransformationCount: 12],
    	PosLength:         [ modelVariant: "inject", operations: [RailwayOperation.POSLENGTH        , RailwayOperation.POSLENGTH_INJECT        ], strategy: TransformationChangeSetStrategy.FIXED, constant: 10, queryTransformationCount: 12],
    	RouteSensor:       [ modelVariant: "inject", operations: [RailwayOperation.ROUTESENSOR      , RailwayOperation.ROUTESENSOR_INJECT      ], strategy: TransformationChangeSetStrategy.FIXED, constant: 10, queryTransformationCount: 12],
    	SemaphoreNeighbor: [ modelVariant: "inject", operations: [RailwayOperation.SEMAPHORENEIGHBOR, RailwayOperation.SEMAPHORENEIGHBOR_INJECT], strategy: TransformationChangeSetStrategy.FIXED, constant: 10, queryTransformationCount: 12],
    	SwitchMonitored:   [ modelVariant: "inject", operations: [RailwayOperation.SWITCHMONITORED  , RailwayOperation.SWITCHMONITORED_INJECT  ], strategy: TransformationChangeSetStrategy.FIXED, constant: 10, queryTransformationCount: 12],
    	SwitchSet:         [ modelVariant: "inject", operations: [RailwayOperation.SWITCHSET        , RailwayOperation.SWITCHSET_INJECT        ], strategy: TransformationChangeSetStrategy.FIXED, constant: 10, queryTransformationCount: 12],
    ]
    
    def runBenchmarkSeries(BenchmarkConfigBaseBuilder configBaseBuilder, BenchmarkConfigBuilder configBuilder,
    		ExecutionConfig ec, ModelSetConfig modelSetConfig) {
    	try {
    		for (def size = modelSetConfig.minSize; size <= modelSetConfig.maxSize; size *= 2) {
    			def modelFilename = "railway-${modelSetConfig.modelVariant}-${size}"
    
    			println("------------------------------------------------------------")
    			println("Model: $modelFilename")
    			println("------------------------------------------------------------")
    
    			configBaseBuilder.setModelFilename(modelFilename)
    			def configBase = configBaseBuilder.createConfigBase()
    			def config = configBuilder.setConfigBase(configBase).createConfig()
    
    			def exitValue = BenchmarkRunner.runPerformanceBenchmark(config, ec)
    			if (exitValue != 0) {
    				println "Timeout or error occured, skipping models for larger sizes. Error code: ${exitValue}"
    				break
    			}
    		}
    	} catch (all) {
    		println "Exception occured during execution."
    	}
    }
    
    workloads.each { workload ->
    	def workloadName = workload.key
    
    	def workloadConfiguration = workload.value
    	def modelVariant = workloadConfiguration["modelVariant"]
    	def operations = workloadConfiguration["operations"]
    	def strategy = workloadConfiguration["strategy"]
    	def constant = workloadConfiguration["constant"]
    	def queryTransformationCount = workloadConfiguration["queryTransformationCount"]
    
    	println("============================================================")
    	println("Workload: $workloadName")
    	println("============================================================")
    
    	def modelSetConfig = new ModelSetConfig(modelVariant, minSize, maxSize)
    
    	def bcbb = new BenchmarkConfigBaseBuilder()
    			.setBenchmarkId(benchmarkId).setTimeout(timeout).setRuns(runs)
    			.setOperations(operations).setWorkload(workloadName)
    			.setQueryTransformationCount(queryTransformationCount).setTransformationConstant(constant)
    			.setTransformationChangeSetStrategy(strategy)
    
    	tools.each{ bcb -> runBenchmarkSeries(bcbb, bcb, ec, modelSetConfig) }
    }
    
    if (binding.variables.get("reportUrl")) {
    	BenchmarkReporter.reportReady(reportUrl)
    }