diff --git a/scripts/run.py b/scripts/run.py index c0db52e3b1a8d15200d4748e7dc681fb96b232ed..2424b7e257ccf42ffe123038aa70742f3eb58a59 100755 --- a/scripts/run.py +++ b/scripts/run.py @@ -52,36 +52,45 @@ def benchmark(conf): shutil.copy(header, result_file) # os.remove(result_file) os.environ['Runs'] = str(conf.Runs) - failed_tools = set() + + # by default we test every model for every tool + available_models = { tool : conf.Models for tool in conf.Tools } + for r in range(0, conf.Runs): os.environ['RunIndex'] = str(r) print("## Run {}".format(r)) + for tool in conf.Tools: config = ConfigParser.ConfigParser() config.read(os.path.join(BASE_DIRECTORY, "solutions", tool, "solution.ini")) set_working_directory("solutions", tool) os.environ['Tool'] = tool - for model in conf.Models: - if not tool in failed_tools: - full_model_path = os.path.abspath(os.path.join(BASE_DIRECTORY, "models", model)) - os.environ['Model'] = model - os.environ['ModelPath'] = full_model_path - print("Running benchmark: tool = " + tool + ", model = " + full_model_path) - try: - output = subprocess.check_output(config.get('run', 'cmd'), shell=True, timeout=conf.Timeout) - with open(result_file, "ab") as file: - file.write(output) - except CalledProcessError as e: - print("Program exited with error") - print('stdout:') - print(e.output) - print('stderr:') - print(e.stderr) - except subprocess.TimeoutExpired as e: - print("Program reached the timeout set ({0} seconds). The command we executed was '{1}'".format(e.timeout, e.cmd)) - failed_tools.add(tool) - else: - print("Skipping tool {0} for model {1} because it already failed earlier".format(tool, model)) + + print('# Available models for {tool} are {models}'.format(tool=tool, models=available_models[tool])) + + for model in available_models[tool]: + full_model_path = os.path.abspath(os.path.join(BASE_DIRECTORY, "models", model)) + os.environ['Model'] = model + os.environ['ModelPath'] = full_model_path + print("Running benchmark: tool = " + tool + ", model = " + full_model_path) + + try: + output = subprocess.check_output(config.get('run', 'cmd'), shell=True, timeout=conf.Timeout) + with open(result_file, "ab") as file: + file.write(output) + except CalledProcessError as e: + print("Program exited with error") + print('stdout:') + print(e.output) + print('stderr:') + print(e.stderr) + except subprocess.TimeoutExpired as e: + print("Program reached the timeout set ({0} seconds). The command we executed was '{1}'".format(e.timeout, e.cmd)) + + # if a tool failed running some model we remove that model as well as all "stronger" models + failed_model_idx = conf.Models.index(model) + available_models[tool] = conf.Models[ : failed_model_idx] + print('# Ignoring models {models} for tool {tool} in the future'.format(models=conf.Models[failed_model_idx : ], tool=tool)) def clean_dir(*path):