invoke.py

#!/usr/bin/env python3

from argparse import ArgumentParser
import os
import sys
import subprocess
import copy
import json
from datetime import datetime
from pathlib import Path

invokePyVersion = '0.2'

treatAsDirectories = ['outputs', 'output', 'outdir', 'dir']

invokeSpec = {}
currentConfigPath = os.path.curdir


def updateBenchSpec(origBenchSpec: dict, source: dict, subdirectory=None):
 benchSpec = copy.deepcopy(origBenchSpec)
 if isinstance(source['dir'], str):
 # If a path is given use it
 benchSpec['dir'] = source['dir'] if os.path.isabs(source['dir']) else os.path.abspath(benchSpec['dir'] + '/' + source['dir'])
 elif source['dir'] is None and subdirectory is not None:
 # If not use the benchmark name as subdirectory
 benchSpec['dir'] = os.path.abspath(benchSpec['dir'] + '/' + subdirectory)
 if isinstance(source['exec'], str):
 benchSpec['exec'] = source['exec'] if os.path.isabs(source['exec']) else os.path.abspath(benchSpec['dir'] + '/' + source['exec'])
 if isinstance(source['stdout'], str):
 benchSpec['stdout'] = source['stdout'] if os.path.isabs(source['stdout']) else os.path.abspath(benchSpec['dir'] + '/' + source['stdout'])
 if isinstance(source['stdout'], str):
 benchSpec['stderr'] = source['stderr'] if os.path.isabs(source['stderr']) else os.path.abspath(benchSpec['dir'] + '/' + source['stderr'])
 if isinstance(source['environment'], dict):
 benchSpec['environment'] = source['environment'] if benchSpec['environment'] is None else {**benchSpec['environment'], **source['environment']}
 if isinstance(source['disabled'], bool):
 benchSpec['disabled'] = source['disabled']

 for k in ['params', 'precmd', 'postcmd']:
 if isinstance(source[k], str):
 benchSpec[k] = source[k]
 elif not source[k]:
 benchSpec[k] = None

 return benchSpec


def batchReplace(target, what: dict, wrapper='%'):
 if isinstance(target, dict):
 for x in target:
 for k, v in what.items():
 target[x] = str(target[x]).replace(wrapper + k + wrapper, v)
 elif isinstance(target, str):
 for k, v in what.items():
 target = target.replace(wrapper + k + wrapper, v)
 return target


def ensureDictionaryKeys(target: dict, keys: list, default=None):
 for k in keys:
 if k not in target:
 target[k] = default


def loadSpecification(spec: dict):
 global invokeSpec
 global currentConfigPath
 if not isinstance(invokeSpec, dict):
 invokeSpec = {}

 # Patching the path of the config file into the specification
 # Resolves relative paths inside the speficiation relative to
 # the config file.
 if isinstance(spec['specs'], list):
 for s in spec['specs']:
 if 'dir' in s and not os.path.isabs(s['dir']):
 s['dir'] = currentConfigPath + '/' + s['dir']
 elif 'dir' not in s:
 s['dir'] = currentConfigPath
 # merge
 for k in spec:
 if isinstance(spec[k], dict):
 if k in invokeSpec and isinstance(invokeSpec[k], dict):
 invokeSpec[k] = {**invokeSpec[k], **spec[k]}
 else:
 invokeSpec[k] = spec[k]
 elif isinstance(spec[k], list):
 if k in invokeSpec and isinstance(invokeSpec[k], list):
 invokeSpec[k].extend(spec[k])
 else:
 invokeSpec[k] = spec[k]
 else:
 invokeSpec[k] = spec[k]


parser = ArgumentParser(description="Invoke and control benchmarks")
parser.add_argument("-c", "--config", help="use this invoke specification", action="append", default=[])
parser.add_argument("-w", "--wrapper", help="Use these invoke wrappers", action="append", default=[])
parser.add_argument("-e", "--environment", help="Use these invoke environments", action="append", default=[])
parser.add_argument("-s", "--suite", help="Invoke these benchmark suites", action="append", default=[])
parser.add_argument("-i", "--input", help="Use these input sets", action="append", default=[])
parser.add_argument("-v", "--variable", help="define varibles e.g. -v var=test", action="append", default=[])
parser.add_argument("-f", "--force", help="ignore benchmark invocations that could not be resolved", action="store_true", default=False)
parser.add_argument("--stdout", help="redirect stdout from benchmark invocation", default=False)
parser.add_argument("--stderr", help="reiderct stderr from benchmark invocation", default=False)
parser.add_argument("--precmd", help="execute this command before each benchmark invocation", default=False)
parser.add_argument("--postcmd", help="execute this command after each benchmark invocation", default=False)
parser.add_argument("--specs", help="Show available benchmarks, suites, wrappers, environments and variables", action="store_true", default=False)
parser.add_argument("--list-benchmarks", help="show a list of specified benchmarks", action="store_true", default=False)
parser.add_argument("--list-suites", help="show a list of specified suites", action="store_true", default=False)

parser.add_argument("--compile", help="compile shell script", action="store_true", default=False)
parser.add_argument("--simulate", help="simulate invocation with verbose output", action="store_true", default=False)
parser.add_argument("--prepare", help="create directories and links", action="store_true", default=False)
parser.add_argument("--verbose", help="verbose output", action="store_true", default=False)
parser.add_argument("--version", help="print version number", action="store_true", default=False)

parser.add_argument("benchmarks", help="Invoke these benchmarks", nargs="*", default=[])

args = parser.parse_args()

if args.version:
 print(f'invoke.py version {invokePyVersion} -- sourced at https://github.com/bgottschall/pythonTools/blob/master/invoke.py')
 exit(0)

if args.simulate:
 print('Simulating, no invocations or changes to the filesystems will be made!')
 args.compile = False
 args.prepare = False

if args.compile:
 args.verbose = False
 args.simulate = False


if len(args.config) == 0:
 if os.path.exists(os.path.curdir + '/' + 'invoke.spec.json'):
 args.config = [os.path.curdir + '/' + 'invoke.spec.json']
 elif os.path.exists(os.path.dirname(__file__) + '/' + 'invoke.spec.json'):
 args.config = [os.path.dirname(__file__) + '/' + 'invoke.spec.json']
 else:
 raise Exception('Could not find any invoke specification files!')

for config in args.config:
 if not os.path.exists(config):
 if args.force:
 if args.verbose:
 print(f"WARNING: could not find invoke specification {config}", file=sys.stderr)
 continue
 raise Exception(f'Invoke specification {args.config} not found!')
 currentConfigPath = os.path.dirname(config)
 try:
 loadSpecification(json.load(open(config)))
 except Exception:
 print(f"Could not parse configuration file {config}", file=sys.stderr)
 raise

if not isinstance(invokeSpec, dict):
 raise Exception('Invalid invoke specification')

benchmarksAvailable = False

# Sanitize Config to make parsing easier
ensureDictionaryKeys(invokeSpec, ['specs', 'variables', 'wrappers', 'environments', 'suites'])
if invokeSpec['specs'] is not None:
 for spec in invokeSpec['specs']:
 ensureDictionaryKeys(spec, ['dir', 'precmd', 'postcmd', 'environment', 'stdout', 'stderr', 'input', 'benchmarks'])
 if spec['benchmarks'] is not None:
 benchmarksAvailable = benchmarksAvailable or len(spec['benchmarks']) > 0
 for b in spec['benchmarks']:
 ensureDictionaryKeys(spec['benchmarks'][b], ['disabled', 'dir', 'exec', 'params', 'precmd', 'postcmd', 'stdout', 'stderr', 'environment', 'input'])
 if spec['benchmarks'][b]['inputs'] is not None:
 for i in spec['benchmarks'][b]['inputs']:
 ensureDictionaryKeys(spec['benchmarks'][b]['inputs'][i], ['disabled', 'dir', 'exec', 'params', 'precmd', 'postcmd', 'stdout', 'stderr', 'environment', 'workloads'])
 if spec['benchmarks'][b]['inputs'][i]['workloads'] is not None:
 for w in spec['benchmarks'][b]['inputs'][i]['workloads']:
 ensureDictionaryKeys(w, ['disabled', 'dir', 'exec', 'params', 'precmd', 'postcmd', 'stdout', 'stderr', 'environment'])

if invokeSpec['variables'] is None:
 invokeSpec['variables'] = {}

if len(args.variable) > 0:
 for v in args.variable:
 vsplit = v.split('=')
 invokeSpec['variables'][vsplit[0]] = '='.join(vsplit[1:])

if args.list_suites:
 if invokeSpec['suites'] is None or len(invokeSpec['suites']) == 0:
 print('No suites specified!')
 else:
 for s in invokeSpec['suites']:
 print(s)
 exit(0)

if args.list_benchmarks:
 if not benchmarksAvailable or invokeSpec['specs'] is None:
 print('No benchmarks specified')
 else:
 if len(args.suite) > 0:
 for s in args.suite:
 if s not in invokeSpec['suites']:
 raise Exception(f"Suite '{s}' not found")
 for b in invokeSpec['suites'][s]['benchmarks']:
 for spec in invokeSpec['specs']:
 if b in spec['benchmarks'] and (not spec['benchmarks'][b]['disabled'] or args.force):
 print(b)
 else:
 for spec in invokeSpec['specs']:
 for b in spec['benchmarks']:
 if not spec['benchmarks'][b]['disabled'] or args.force:
 print(b)
 exit(0)


if args.specs:
 print(f"{'Benchmark':24s} {'Inputs'}")
 print('---')
 if not benchmarksAvailable or invokeSpec['specs'] is None:
 print('No benchmarks specified')
 else:
 for spec in invokeSpec['specs']:
 for b in spec['benchmarks']:
 print(f"{b:24s} {', '.join(spec['benchmarks'][b]['inputs'].keys())}{' (disabled)' if spec['benchmarks'][b]['disabled'] else ''}")
 print('')
 print(f"{'Suite':24s} {'Benchmarks'}")
 print('---')
 if invokeSpec['suites'] is None or len(invokeSpec['suites']) == 0:
 print('No suites specified!')
 else:
 for s in invokeSpec['suites']:
 print(f"{s:24s} {', '.join(invokeSpec['suites'][s]['benchmarks'])}")
 print('')
 print(f"{'Wrapper':24s} {'Definition'}")
 print('---')
 if invokeSpec['wrappers'] is None or len(invokeSpec['wrappers']) == 0:
 print('No wrappers specified')
 else:
 for w in invokeSpec['wrappers']:
 print(f"{w:24s} {invokeSpec['wrappers'][w]}")
 print('')
 print(f"{'Variable':24s} {'Definition'}")
 print('---')
 if invokeSpec['variables'] is None or len(invokeSpec['variables']) == 0:
 print('No variables specified')
 else:
 for v in invokeSpec['variables']:
 print(f"{v:24s} {invokeSpec['variables'][v]}")
 print('')

 print(f"{'Environment':24s} {'Definition'}")
 print('---')
 if invokeSpec['environments'] is None or len(invokeSpec['environments']) == 0:
 print('No environments specified')
 else:
 for e in invokeSpec['environments']:
 print(f"{e:24s} {invokeSpec['environments'][e]}")
 exit(0)


if not benchmarksAvailable:
 raise Exception('No benchmarks are specified in the configuration!')

for suite in args.suite:
 if suite not in invokeSpec['suites']:
 raise Exception(f"Suite '{suite}' not found")
 if isinstance(invokeSpec['suites'][suite]['benchmarks'], list):
 args.benchmarks.extend(invokeSpec['suites'][suite]['benchmarks'])
 else:
 args.benchmarks.append(invokeSpec['suites'][suite]['benchmarks'])

# create a unique benchmark selection, no need to invoke a benchmark more than once (no logical for this script)
uniqueBenchmarks = set()
args.benchmarks = [b for b in args.benchmarks if b not in uniqueBenchmarks and not uniqueBenchmarks.add(b)]
del uniqueBenchmarks

if len(args.benchmarks) == 0:
 print('No benchmarks selected for invocation!')
 exit(1)


# Make sure its all string
for v in invokeSpec['variables']:
 invokeSpec['variables'][v] = str(invokeSpec['variables'][v])


# Lets build the default Invoke CMD line
defaultInvoke = ''

if len(args.wrapper) > 0:
 for w in args.wrapper:
 if invokeSpec['wrappers'] is None or w not in invokeSpec['wrappers']:
 raise Exception(f"Wrapper '{w}' not found!")
 defaultInvoke += invokeSpec['wrappers'][w]
 if not defaultInvoke.endswith(' '):
 defaultInvoke += ' '


# Start constructing the environment
globalEnvironment = {}
globalVarEnvironment = {}

if len(args.environment) > 0:
 for e in args.environment:
 if invokeSpec['environments'] is None or e not in invokeSpec['environments']:
 raise Exception(f"Environment '{e}' not found!")
 for k, v in invokeSpec['environments'][e].items():
 v = str(v)
 if v.count('%') >= 2:
 globalVarEnvironment[k] = v
 else:
 globalEnvironment[k] = v

globalVarEnvironment = batchReplace(globalVarEnvironment, invokeSpec['variables'])
tempDict = {}
for k, v in globalVarEnvironment.items():
 if v.count('%') < 2:
 globalEnvironment[k] = v
 else:
 tempDict[k] = v
globalVarEnvironment = tempDict


if args.compile:
 shellScript = '#!/bin/sh\n'
 shellScript += '\n'
 shellScript += 'PWDPATH="$(pwd -P)"\n'
 shellScript += 'SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"\n'
 shellScript += '\n'
 shellScript += '[ "$PWDPATH" != "$SCRIPTPATH" ] && cd "$SCRIPTPATH"\n'
 shellScript += '\n'

if args.compile and len(globalEnvironment) > 0:
 shellScript += 'export '
 for k in globalEnvironment:
 shellScript += f'{k}={globalEnvironment[k]} '
 shellScript += '\n'

if args.simulate and len(globalEnvironment) > 0:
 print('/:$ export ', end='')
 for k in globalEnvironment:
 print(f'{k}={globalEnvironment[k]} ', end='')
 print('')

for d in treatAsDirectories:
 if d in invokeSpec['variables']:
 if not os.path.exists(invokeSpec['variables'][d]):
 if args.prepare or (not args.compile and not args.simulate):
 try:
 Path(invokeSpec['variables'][d]).mkdir(parents=True, exist_ok=True)
 except Exception:
 pass
 elif args.compile:
 shellScript += f"mkdir -p {invokeSpec['variables'][d]} 2>/dev/null || true\n"
 if args.simulate:
 print(f"/:$ mkdir -p {invokeSpec['variables'][d]} || true")

if args.compile:
 shellScript += '\n'

invokeCounter = 0
failedInvokes = []
outputFiles = []

for benchmark in args.benchmarks:
 benchmarkFound = False
 for specIndex, spec in enumerate(invokeSpec['specs']):
 if benchmark not in spec['benchmarks']:
 continue

 benchmarkFound = True
 subEnvironment = {}

 benchSpecL0 = {
 'dir': spec['dir'] if spec['dir'] is not None else os.path.curdir,
 'exec': None,
 'params': None,
 'precmd': spec['precmd'],
 'postcmd': spec['postcmd'],
 'stdout': spec['stdout'],
 'stderr': spec['stderr'],
 'environment': {},
 'disabled': False,
 }

 if isinstance(spec['environment'], dict):
 for k, v in spec['environment'].items():
 v = str(v)
 if v.count('%') >= 2:
 v = batchReplace(v, invokeSpec['variables'])
 if v.count('%') >= 2:
 benchSpecL0['environment'][k] = v
 else:
 subEnvironment[k] = v

 if args.compile and len(subEnvironment) > 0:
 shellScript += 'export '
 for k in subEnvironment:
 shellScript += f'{k}={subEnvironment[k]} '
 shellScript += '\n'

 if args.simulate and len(subEnvironment) > 0:
 print('/:$ export ', end='')
 for k in subEnvironment:
 print(f'{k}={subEnvironment[k]} ', end='')
 print('')

 if len(args.input) == 0:
 if spec['input'] is None:
 if args.force:
 if args.verbose:
 print(f'WARNING: specification {specIndex} odes not provide a default input, skipping.', file=sys.stderr)
 continue
 raise Exception(f"Specficiation {specIndex} does not have a default input, please specify one!")
 useInputs = spec['input'] if isinstance(spec['input'], list) else [spec['input']]
 else:
 useInputs = args.input

 benchSpecL1 = updateBenchSpec(benchSpecL0, spec['benchmarks'][benchmark], benchmark)

 for input in useInputs:
 if input not in spec['benchmarks'][benchmark]['inputs']:
 print(f"WARNING: Could not find input '{input}' for benchmark '{benchmark}', will skip invocation...", file=sys.stderr)
 continue

 benchSpecL2 = updateBenchSpec(benchSpecL1, spec['benchmarks'][benchmark]['inputs'][input], input)

 if len(spec['benchmarks'][benchmark]['inputs'][input]['workloads']) == 0:
 print(f"WARNING: no workloads defined for benchmark '{benchmark}', will skip invocation...", file=sys.stderr)
 continue

 symlinked = False

 for workload, _ in enumerate(spec['benchmarks'][benchmark]['inputs'][input]['workloads']):
 benchSpec = updateBenchSpec(benchSpecL2, spec['benchmarks'][benchmark]['inputs'][input]['workloads'][workload])

 # the %now% variable is replaced by the datetime, if compiling its resolved through the shell

 if benchSpec['disabled']:
 if args.force:
 print(f"WARNING: ignore disabled flag for benchmark '{benchmark}'", file=sys.stderr)
 else:
 if args.verbose:
 print(f"Ignore disabled benchmark '{benchmark}'")
 continue

 if benchSpec['exec'] is None:
 if args.force:
 if args.verbose:
 print(f"WARNING: ignored workload {workload} of '{benchmark}' because no executable was defined", file=sys.stderr)
 continue
 raise Exception(f"No executable defined for benchmark '{benchmark}'")

 if benchSpec['params'] is None:
 benchSpec['params'] = ''

 if args.precmd:
 benchSpec['precmd'] = args.precmd
 if args.postcmd:
 benchSpec['postcmd'] = args.postcmd

 if args.compile and (('%now%' in str(benchSpec)) or ('%now%' in defaultInvoke)):
 sDate = '${NOW}'
 else:
 sDate = datetime.now().strftime("%Y-%m-%d_%H%M%S")

 replaceVars = {**{'counter': str(invokeCounter), 'workload': str(workload), 'input': str(input), 'benchmark': str(benchmark), 'now': sDate}, **invokeSpec['variables']}

 benchSpec['dir'] = batchReplace(benchSpec['dir'], replaceVars)
 benchSpec['exec'] = batchReplace(benchSpec['exec'], replaceVars)

 execName = os.path.basename(benchSpec['exec'])

 benchSpec['exec'] = os.path.relpath(benchSpec['exec'], os.path.curdir)
 if not os.path.isdir(benchSpec['dir']):
 if args.verbose:
 print(f"Creating directory '{benchSpec['dir']}'")
 if args.simulate:
 print(f"/:$ mkdir -p {benchSpec['dir']}")
 else:
 Path(benchSpec['dir']).mkdir(parents=True, exist_ok=True)

 # Executable is not where it is supposed to be, but one is available in the input directory
 if not os.path.exists(benchSpec['exec']) and os.path.exists(benchSpec['dir'] + '/' + execName):
 if args.verbose:
 print(f"WARNING: couldn't find {benchSpec['exec']}, will use {benchSpec['dir'] + '/' + execName} instead", file=sys.stderr)
 benchSpec['exec'] = benchSpec['dir'] + '/' + execName

 if not os.path.exists(benchSpec['exec']):
 if args.force:
 if args.verbose:
 print(f"WARNING: ignored workload {workload} of '{benchmark}' because executable '{benchSpec['exec']}' was not found", file=sys.stderr)
 continue
 raise Exception(f"Could not find executable '{benchSpec['exec']}'")

 benchSpec['dir'] = os.path.relpath(benchSpec['dir'], os.path.curdir)

 if args.compile:
 shellScript += f"# Execute workload {workload} of the '{input}' input of benchmark '{benchmark}'\n"
 if sDate == '${NOW}':
 shellScript += 'NOW="$(date +\'%Y-%m-%d_%H%M%S\')"\n'

 if args.verbose and not args.prepare:
 print(f"Executing benchmark '{benchmark}', input '{input}', workload {workload}")

 if not os.path.exists(benchSpec['dir'] + '/' + execName):
 if args.verbose:
 print(f"Symlinking '{benchSpec['exec']}' to '{benchSpec['dir'] + '/' + execName}'")
 if args.simulate:
 print(f"/:$ ln -s {os.path.relpath(benchSpec['exec'], benchSpec['dir'])} {benchSpec['dir'] + '/' + execName}")
 elif args.prepare or not args.compile:
 os.symlink(os.path.relpath(benchSpec['exec'], benchSpec['dir']), benchSpec['dir'] + '/' + execName)
 else:
 shellScript += f"ln -s {os.path.relpath(benchSpec['exec'], benchSpec['dir'])} {benchSpec['dir'] + '/' + execName}\n"
 symlinked = True
 elif os.path.realpath(benchSpec['exec']) != os.path.realpath(benchSpec['dir'] + '/' + execName):
 print(f"WARNING: target executable '{benchSpec['dir'] + '/' + execName}' differs from specified executable '{benchSpec['exec']}'", file=sys.stderr)

 invokeCmd = defaultInvoke + './' + execName
 if len(benchSpec['params']) > 0:
 invokeCmd += ' ' + benchSpec['params']

 benchSpec['environment'] = {**benchSpec['environment'], **globalVarEnvironment}

 replaceVars = {**replaceVars, **{'dir': benchSpec['dir'], 'exec': execName}}

 for d in treatAsDirectories:
 if d in replaceVars:
 replaceVars[d] = os.path.abspath(replaceVars[d]) + '/' if os.path.isabs(replaceVars[d]) else os.path.relpath(replaceVars[d], benchSpec['dir']) + '/'

 # Postprocess the invoke cmd lines after variables
 invokeCmd = batchReplace(invokeCmd, replaceVars)
 if isinstance(benchSpec['precmd'], str):
 benchSpec['precmd'] = batchReplace(benchSpec['precmd'], replaceVars)
 if isinstance(benchSpec['postcmd'], str):
 benchSpec['postcmd'] = batchReplace(benchSpec['postcmd'], replaceVars)

 if args.stdout:
 benchSpec['stdout'] = args.stdout if os.path.isabs(args.stdout) else os.path.relpath(args.stdout, benchSpec['dir'])
 if args.stderr:
 benchSpec['stderr'] = args.stderr if os.path.isabs(args.stderr) else os.path.relpath(args.stderr, benchSpec['dir'])

 if isinstance(benchSpec['stdout'], str):
 benchSpec['stdout'] = batchReplace(benchSpec['stdout'], replaceVars)

 if isinstance(benchSpec['stderr'], str):
 benchSpec['stderr'] = batchReplace(benchSpec['stderr'], replaceVars)

 benchSpec['environment'] = batchReplace(benchSpec['environment'], replaceVars)

 if not args.compile:
 invokeEnvironment = {**os.environ.copy(), **subEnvironment, **benchSpec['environment'], **globalEnvironment}

 if benchSpec['stdout'] is not None:
 # Append to invokeCmd or open it for redirected output
 if args.compile or args.simulate:
 invokeCmd += f" >>{benchSpec['stdout']}"
 else:
 benchSpec['stdout'] = open(benchSpec['stdout'] if os.path.isabs(benchSpec['stdout']) else benchSpec['dir'] + '/' + benchSpec['stdout'], 'a')

 if benchSpec['stderr'] is not None:
 # Append to invokeCmd or open it for redirected output
 if args.compile or args.simulate:
 invokeCmd += f" 2>>{benchSpec['stderr']}"
 else:
 benchSpec['stderr'] = open(benchSpec['stderr'] if os.path.isabs(benchSpec['stderr']) else benchSpec['dir'] + '/' + benchSpec['stderr'], 'a')

 if args.compile:
 shellScript += '(\n'
 shellScript += ' set -x\n'
 shellScript += f" cd \"{benchSpec['dir']}\"\n"
 if len(benchSpec['environment']) > 0:
 shellScript += ' export '
 for k in benchSpec['environment']:
 shellScript += f"{k}={benchSpec['environment'][k]} "
 shellScript += '\n'
 if args.simulate:
 if len(benchSpec['environment']) > 0:
 print(f"{benchSpec['dir']}:$ export ", end='')
 for k in benchSpec['environment']:
 print(f"{k}={benchSpec['environment'][k]} ", end='')
 print('')
 if args.verbose and len(benchSpec['environment']) > 0:
 print(f"Setting environment to {benchSpec['environment']}")

 if isinstance(benchSpec['precmd'], str):
 if not args.prepare:
 if args.verbose:
 print(f"Executing pre invoke command '{benchSpec['precmd']}'")
 if args.simulate:
 print(f"{benchSpec['dir']}:$ {benchSpec['precmd']}")
 elif not args.prepare and not args.compile:
 ret = subprocess.call(benchSpec['precmd'], shell=True, cwd=benchSpec['dir'], env=invokeEnvironment)
 if ret != 0:
 if args.verbose:
 print(f"Execution failed with return code {ret}")
 failedInvokes.append(f"{benchmark}-{input}-{workload}-precmd")
 if args.compile:
 shellScript += f" {benchSpec['precmd']}\n"

 if not args.prepare and args.verbose:
 print(f"Invoke command line '{invokeCmd}'")
 if not args.compile and not args.simulate:
 if benchSpec['stdout'] is not None:
 print(f"Redirect stdout to {benchSpec['stdout'].name}")
 if benchSpec['stderr'] is not None:
 print(f"Redirect stderr to {benchSpec['stderr'].name}")
 if args.simulate:
 print(f"{benchSpec['dir']}:$ {invokeCmd}")
 elif not args.prepare and not args.compile:
 ret = subprocess.call(invokeCmd, shell=True, cwd=benchSpec['dir'], env=invokeEnvironment, stdout=benchSpec['stdout'], stderr=benchSpec['stderr'])
 if ret != 0:
 if args.verbose:
 print(f"Execution failed with return code {ret}")
 failedInvokes.append(f"{benchmark}-{input}-{workload}")
 if args.compile:
 shellScript += f" {invokeCmd}\n"

 if isinstance(benchSpec['postcmd'], str):
 if not args.prepare:
 if args.verbose:
 print(f"Executing post invoke command '{benchSpec['postcmd']}'")
 if args.simulate:
 print(f"{benchSpec['dir']}:$ {benchSpec['postcmd']}")
 elif not args.compile:
 ret = subprocess.call(benchSpec['postcmd'], shell=True, cwd=benchSpec['dir'], env=invokeEnvironment)
 if ret != 0:
 if args.verbose:
 print(f"Execution failed with return code {ret}")
 failedInvokes.append(f"{benchmark}-{input}-{workload}-postcmd")
 if args.compile:
 shellScript += f" {benchSpec['postcmd']}\n"

 invokeCounter += 1

 if args.compile:
 shellScript += ')\n'

 if symlinked and not args.prepare:
 if args.verbose:
 print("Remove previously created symlink")
 if args.simulate:
 print(f"/:$ rm {benchSpec['dir'] + '/' + execName}")
 elif not args.compile:
 os.unlink(benchSpec['dir'] + '/' + execName)
 if args.compile:
 shellScript += f"rm {benchSpec['dir'] + '/' + execName}\n"

 if args.compile:
 shellScript += '\n'

 if not benchmarkFound:
 raise Exception(f"Could not find specification for benchmark '{benchmark}'!")

if args.compile:
 print(shellScript, end='')

if len(failedInvokes) != 0:
 print(f'WARNING: detected {failedInvokes} with an error return code!', file=sys.stderr)