Skip to main content
Explore
Sign in
Explore
Snippets Groups Projects
Commit 100d06e2 authored by René Schöne's avatar René Schöne
Browse files

Added new scenario (bigger, mixed kinds). 

- notebook: customizing diagram titles
- added start, end times
parent 1f81cabd
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
ilp-measurement.ipynb
+ 170
− 185
View file @ 100d06e2
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import numpy as np import numpy as np
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import matplotlib.axes, matplotlib.ticker import matplotlib.axes, matplotlib.ticker
%matplotlib inline %matplotlib inline
plt.ioff() plt.ioff()
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
from glob import glob from glob import glob
from datetime import datetime from datetime import datetime
from itertools import * from itertools import *
from operator import itemgetter from operator import itemgetter
import json, sys, traceback, os import json, sys, traceback, os, functools
```
%% Cell type:code id: tags:
``` python
if not os.path.exists('doc'):
os.mkdir('doc')
if not os.path.exists('pngs'):
os.mkdir('pngs')
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
spec_files = glob('profiling/*/specs') spec_files = glob('profiling/*/specs')
spec_files.sort() spec_files.sort()
specs = {os.path.basename(os.path.dirname(f)):np.genfromtxt(f, delimiter=' ', dtype=int) specs = {os.path.basename(os.path.dirname(f)):np.genfromtxt(f, delimiter=' ', dtype=int)
for f in spec_files} for f in spec_files}
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
linestyles = ["-","--","-.",":"] linestyles = ["-","--","-.",":"]
colors = ('blue','black','red','orange','green','magenta','yellow', 'cyan', 'purple', 'firebrick') colors = ('blue','black','red','orange','green','magenta','yellow', 'cyan', 'purple', 'firebrick')
colorcycle = cycle(colors) colorcycle = cycle(colors)
lastpe = False lastpe = False
line_def = [] line_def = []
color_def = [] color_def = []
## Currently disabled due to fragile and unused output ## Currently disabled due to fragile and unused output
if False: if False:
for spec in specs: for spec in specs:
pe, comp, impl, mode = (spec[2], spec[4], spec[5], spec[6]) pe, comp, impl, mode = (spec[2], spec[4], spec[5], spec[6])
if not lastpe or lastpe != pe: if not lastpe or lastpe != pe:
color = next(colorcycle) color = next(colorcycle)
linecycle = cycle(linestyles) linecycle = cycle(linestyles)
line = next(linecycle) line = next(linecycle)
lastpe = pe lastpe = pe
line_def.append(line) line_def.append(line)
color_def.append(color) color_def.append(color)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def set_keys(*indices): def set_keys(*indices):
"""Returns a function that returns a tuple of key values""" """Returns a function that returns a tuple of key values"""
def get_keys(seq, indices=indices): def get_keys(seq, indices=indices):
keys = [] keys = []
for i in indices: for i in indices:
keys.append(seq[i]) keys.append(seq[i])
return tuple(keys) return tuple(keys)
return get_keys return get_keys
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def get_average_times(dat, dirCol, stepCol, timeCol): def get_average_times(dat, dirCol, stepCol, timeCol):
if dat.size == 1: if dat.size == 1:
return np.array([np.array([dat.item()[timeCol]])]) return np.array([np.array([dat.item()[timeCol]])])
dat.sort(order=['dir', 'step']) dat.sort(order=['dir', 'step'])
result = {} result = {}
for (c_dir, c_step), rows in groupby(dat, key=set_keys('dir','step')): for (c_dir, c_step), rows in groupby(dat, key=set_keys('dir','step')):
#print c_dir, c_step, rows #print c_dir, c_step, rows
c_dir, c_step, total_time, counter = None, None, 0, 0 c_dir, c_step, total_time, counter = None, None, 0, 0
for row in rows: for row in rows:
if not c_dir: if not c_dir:
c_dir = row[dirCol] c_dir = row[dirCol]
if not c_step: if not c_step:
c_step = row[stepCol] c_step = row[stepCol]
total_time+=row[timeCol] total_time+=row[timeCol]
counter += 1 counter += 1
result.setdefault(c_dir, []).append([c_step, total_time*1.0/counter]) result.setdefault(c_dir, []).append([c_step, total_time*1.0/counter])
result2 = [] result2 = []
for c_dir, rows in result.iteritems(): for c_dir, rows in result.iteritems():
inner = [] inner = []
result2.append([row[1] for row in rows]) result2.append([row[1] for row in rows])
# for row in rows: # for row in rows:
# inner.append(row[1]) # inner.append(row[1])
# result2.append(inner) # result2.append(inner)
return np.array([np.array(rows) for rows in result2]) return np.array([np.array(rows) for rows in result2])
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def is_axis(a): def is_axis(a):
return not type(a) is np.ndarray return not type(a) is np.ndarray
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
with open('profiling/kinds.json') as fd: with open('profiling/kinds.json') as fd:
kinds = json.load(fd) kinds = json.load(fd)
print 'Kinds: {}'.format(kinds) print 'Kinds: {}'.format(kinds)
``` ```
%% Output %% Output
Kinds: {u'strategies': [u'normal', u'flush', u'noncached'], u'changes': [u'update', u'sw', u'res', u'complex']} Kinds: {u'strategies': [u'normal', u'flush', u'noncached'], u'changes': [u'update', u'sw', u'res', u'complex', u'mixed']}
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
no_split, split_change_only, split_both = 0, 1, 2 no_split, split_change_only, split_both = 0, 1, 2
dat, dat2, dat3 = None, None, None dat, dat2, dat3 = None, None, None
def read_single_result(f, name, dtype, data_column, since): def read_single_result(f, name, dtype, data_column, since):
global dat, dat2, dat3 global dat, dat2, dat3
def convdate(text): def convdate(text):
return datetime.strptime(text, '%Y-%m-%dT%H:%M:%S.%f') return datetime.strptime(text, '%Y-%m-%dT%H:%M:%S.%f')
def convdir(text): def convdir(text):
return int(text[-3:]) return int(text[-3:])
def convstep(text): def convstep(text):
return int(text[0:2]) return int(text[0:2])
def safe_div(a, b): def safe_div(a, b):
return a/b if b>0 else "{}!".format(a) return a/b if b>0 else "{}!".format(a)
dat = np.genfromtxt(f, delimiter=',', names=True, dtype=dtype, dat = np.genfromtxt(f, delimiter=',', names=True, dtype=dtype,
converters={'timestamp':convdate, 'step':convstep, 'dir': convdir}) converters={'timestamp':convdate, 'step':convstep, 'dir': convdir})
if since: if since:
dat = dat[dat['timestamp'] > since ] dat = dat[dat['timestamp'] > since ]
dat2 = get_average_times(dat, 1, 2, data_column).transpose() dat2 = get_average_times(dat, 1, 2, data_column).transpose()
len_dat = 1 if len(dat.shape) == 0 else len(dat) len_dat = 1 if len(dat.shape) == 0 else len(dat)
if dat2.size == 0: if dat2.size == 0:
print 'Did not load any record for {}'.format(name) print 'Did not load any record for {}'.format(name)
else: else:
runs = '~= {} run(s)'.format(safe_div(len_dat,dat2.size)) if len_dat > 1 else '- no data' if len_dat > 1:
print 'Loaded {0} record(s) for {1} ({2[0]}x{2[1]} unique) {3}'.format(len_dat, name, dat2.shape, runs) print 'Loaded {0} record(s) for {1} ({2[0]}x{2[1]} unique) ~= {3} run(s)'.format(
len_dat, name, dat2.shape, safe_div(len_dat,dat2.size))
else:
print 'Loaded no data for {}'.format(name)
dat3 = dat2 dat3 = dat2
return dat2 return dat2
def read_results(prefix, name, dtype, data_column, since, splitted = no_split): def read_results(prefix, name, dtype, data_column, since, splitted = no_split):
if splitted in (split_change_only, split_both): if splitted in (split_change_only, split_both):
result = {} result = {}
for change in kinds['changes']: for change in kinds['changes']:
result.setdefault(change, {}) result.setdefault(change, {})
if splitted == split_both: if splitted == split_both:
for strategy in kinds['strategies']: for strategy in kinds['strategies']:
result[change].setdefault(strategy, {}) result[change].setdefault(strategy, {})
new_name = '{0}_{1}_{2}'.format(change, strategy, name) new_name = '{0}_{1}_{2}'.format(change, strategy, name)
f = 'profiling/splitted/{0}_{1}.csv'.format(prefix, new_name) f = 'profiling/splitted/{0}_{1}.csv'.format(prefix, new_name)
result[change][strategy] = read_single_result(f, new_name, dtype, data_column, since) result[change][strategy] = read_single_result(f, new_name, dtype, data_column, since)
else: # splitted == split_change_only else: # splitted == split_change_only
new_name = '{0}_{1}'.format(change, name) new_name = '{0}_{1}'.format(change, name)
f = 'profiling/splitted/{0}_{1}.csv'.format(prefix, new_name) f = 'profiling/splitted/{0}_{1}.csv'.format(prefix, new_name)
result[change] = read_single_result(f, new_name, dtype, data_column, since) result[change] = read_single_result(f, new_name, dtype, data_column, since)
return result return result
else: # splitted = no_split else: # splitted = no_split
f = 'profiling/{0}-{1}-results.csv'.format(prefix, name) f = 'profiling/{0}-{1}-results.csv'.format(prefix, name)
return read_single_result(f, name, dtype, data_column, since) return read_single_result(f, name, dtype, data_column, since)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def read_gen_results(impl, since = None): def read_gen_results(impl, since = None):
return read_results('gen', impl, ('datetime64[us]', int, int, float), 3, since, splitted = split_both) return read_results('gen', impl, ('datetime64[us]', int, int, float), 3, since, splitted = split_both)
def read_sol_results(solver, since = None): def read_sol_results(solver, since = None):
return read_results('sol', solver, ('datetime64[us]', int, int, int, int, int, float, float), 7, since, return read_results('sol', solver, ('datetime64[us]', int, int, int, int, int, float, float), 7, since,
splitted = split_change_only) splitted = split_change_only)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def safe(a, i, start = 0): def safe(a, i, start = 0):
try: try:
return a[start:,i] return a[start:,i]
except IndexError: except IndexError:
return np.zeros(a[:,0].size) return np.zeros(a[:,0].size)
def es(y, x): def es(y, x):
""" Ensure size of y compared to x """ """ Ensure size of y compared to x """
if y.shape[0] != x.shape[0]: if y.shape[0] != x.shape[0]:
y = np.ones(x.shape[0]) * y[0] y = np.ones(x.shape[0]) * y[0]
return y return y
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## ILP-Re-Generation and Solving Time ## ILP-Re-Generation and Solving Time
- ILP-Re-Generation describes the time it takes for the Scheme implementations to evaluate the attribute computing the ILP - ILP-Re-Generation describes the time it takes for the Scheme implementations to evaluate the attribute computing the ILP
- it does not include the time to read from and write to disk - it does not include the time to read from and write to disk
- "Java" denotes the Java- and EMF-based generation - "Java" denotes the Java- and EMF-based generation
- ILP Solving describes the time it takes the solvers (GLPK and Gurobi in this case) to solve the generated ILP - ILP Solving describes the time it takes the solvers (GLPK and Gurobi in this case) to solve the generated ILP
- "GLPK (Java)" denotes the time to solve the ILP generated by the Java-based generation with GLPK. Its format was modifiert slightly to be accepted by GLPK, as it was originally generated for lp_solve - "GLPK (Java)" denotes the time to solve the ILP generated by the Java-based generation with GLPK. Its format was modifiert slightly to be accepted by GLPK, as it was originally generated for lp_solve
- "GLPK (Scheme)" denotes the time to solve the ILP generated by any scheme implementation with GLPK - "GLPK (Scheme)" denotes the time to solve the ILP generated by any scheme implementation with GLPK
- the plots show different system configurations - the plots show different system configurations
- a system configuration is given by "r x ( c \* i \* m )", which describes a system with *r* resources and *c* software components with *i* implementations having *m* modes each. - a system configuration is given by "r x ( c \* i \* m )", which describes a system with *r* resources and *c* software components with *i* implementations having *m* modes each.
- for one such configuration the same experiment is run, i.e. the system is modified 7 times leading to the 7 steps, whereas only changes on hardware resources are made - for one such configuration the same experiment is run, i.e. the system is modified 7 times leading to the 7 steps, whereas only changes on hardware resources are made
- for the cases involving Java (both, generation and solving), the same time is used for every step, as the generation always starts from skretch. Further, the changes (e.g. change the value of a single hardware resource while generation) can not be reflected by the Java System Generator - for the cases involving Java (both, generation and solving), the same time is used for every step, as the generation always starts from skretch. Further, the changes (e.g. change the value of a single hardware resource while generation) can not be reflected by the Java System Generator
- on the x-axis, the steps of this manipulation are shown - on the x-axis, the steps of this manipulation are shown
- the initial generation of the ILP (step zero) is only shown below in numbers, as it would skew the diagrams - the initial generation of the ILP (step zero) is only shown below in numbers, as it would skew the diagrams
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
Header = ['timestamp', 'dir', 'step', 'rows', 'cols', 'non-zero', 'ilp-sol', 'ti-ilp-sol'] Header = ['timestamp', 'dir', 'step', 'rows', 'cols', 'non-zero', 'ilp-sol', 'ti-ilp-sol']
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
racket_dats = read_gen_results('plt-r6rs', since = datetime(2015,8,17,0,0,0)) racket_dats = read_gen_results('plt-r6rs', since = datetime(2015,8,17,0,0,0))
larceny_dats = read_gen_results('larceny', since = datetime(2015,6,12,0,0,0)) larceny_dats = read_gen_results('larceny', since = datetime(2015,6,12,0,0,0))
java_dats = read_gen_results('java') java_dats = read_gen_results('java')
``` ```
%% Output %% Output
Loaded 1320 record(s) for update_normal_plt-r6rs (8x31 unique) ~= 5 run(s) Loaded 1320 record(s) for update_normal_plt-r6rs (8x31 unique) ~= 5 run(s)
Loaded 504 record(s) for update_flush_plt-r6rs (8x31 unique) ~= 2 run(s) Loaded 504 record(s) for update_flush_plt-r6rs (8x31 unique) ~= 2 run(s)
Loaded 600 record(s) for update_noncached_plt-r6rs (8x31 unique) ~= 2 run(s) Loaded 600 record(s) for update_noncached_plt-r6rs (8x31 unique) ~= 2 run(s)
Loaded 364 record(s) for sw_normal_plt-r6rs (7x31 unique) ~= 1 run(s) Loaded 364 record(s) for sw_normal_plt-r6rs (7x31 unique) ~= 1 run(s)
Loaded 245 record(s) for sw_flush_plt-r6rs (7x31 unique) ~= 1 run(s) Loaded 245 record(s) for sw_flush_plt-r6rs (7x31 unique) ~= 1 run(s)
Loaded 602 record(s) for sw_noncached_plt-r6rs (7x31 unique) ~= 2 run(s) Loaded 602 record(s) for sw_noncached_plt-r6rs (7x31 unique) ~= 2 run(s)
Loaded 360 record(s) for res_normal_plt-r6rs (7x31 unique) ~= 1 run(s) Loaded 360 record(s) for res_normal_plt-r6rs (7x31 unique) ~= 1 run(s)
Loaded 245 record(s) for res_flush_plt-r6rs (7x31 unique) ~= 1 run(s) Loaded 245 record(s) for res_flush_plt-r6rs (7x31 unique) ~= 1 run(s)
Loaded 392 record(s) for res_noncached_plt-r6rs (7x31 unique) ~= 1 run(s) Loaded 609 record(s) for res_noncached_plt-r6rs (7x31 unique) ~= 2 run(s)
Loaded 242 record(s) for complex_normal_plt-r6rs (11x11 unique) ~= 2 run(s) Loaded 242 record(s) for complex_normal_plt-r6rs (11x11 unique) ~= 2 run(s)
Loaded 1 record(s) for complex_flush_plt-r6rs (1x1 unique) - no data Loaded 121 record(s) for complex_flush_plt-r6rs (11x11 unique) ~= 1 run(s)
Loaded 1 record(s) for complex_noncached_plt-r6rs (1x1 unique) - no data Loaded 121 record(s) for complex_noncached_plt-r6rs (11x11 unique) ~= 1 run(s)
Loaded 264 record(s) for update_normal_larceny (8x31 unique) ~= 1 run(s) Loaded 200 record(s) for mixed_normal_plt-r6rs (100x2 unique) ~= 1 run(s)
Loaded 1 record(s) for update_flush_larceny (1x1 unique) - no data Loaded no data for mixed_flush_plt-r6rs
Loaded 1 record(s) for update_noncached_larceny (1x1 unique) - no data Loaded no data for mixed_noncached_plt-r6rs
Loaded 1 record(s) for sw_normal_larceny (1x1 unique) - no data Loaded 512 record(s) for update_normal_larceny (8x31 unique) ~= 2 run(s)
Loaded 1 record(s) for sw_flush_larceny (1x1 unique) - no data Loaded no data for update_flush_larceny
Loaded 1 record(s) for sw_noncached_larceny (1x1 unique) - no data Loaded no data for update_noncached_larceny
Loaded 1 record(s) for res_normal_larceny (1x1 unique) - no data Loaded 217 record(s) for sw_normal_larceny (7x31 unique) ~= 1 run(s)
Loaded 1 record(s) for res_flush_larceny (1x1 unique) - no data Loaded no data for sw_flush_larceny
Loaded 1 record(s) for res_noncached_larceny (1x1 unique) - no data Loaded no data for sw_noncached_larceny
Loaded 1 record(s) for complex_normal_larceny (1x1 unique) - no data Loaded 217 record(s) for res_normal_larceny (7x31 unique) ~= 1 run(s)
Loaded 1 record(s) for complex_flush_larceny (1x1 unique) - no data Loaded no data for res_flush_larceny
Loaded 1 record(s) for complex_noncached_larceny (1x1 unique) - no data Loaded no data for res_noncached_larceny
Loaded 121 record(s) for complex_normal_larceny (11x11 unique) ~= 1 run(s)
Loaded no data for complex_flush_larceny
Loaded no data for complex_noncached_larceny
Loaded no data for mixed_normal_larceny
Loaded no data for mixed_flush_larceny
Loaded no data for mixed_noncached_larceny
Loaded 88 record(s) for update_normal_java (1x31 unique) ~= 2 run(s) Loaded 88 record(s) for update_normal_java (1x31 unique) ~= 2 run(s)
Loaded 1 record(s) for update_flush_java (1x1 unique) - no data Loaded no data for update_flush_java
Loaded 1 record(s) for update_noncached_java (1x1 unique) - no data Loaded no data for update_noncached_java
Loaded 1 record(s) for sw_normal_java (1x1 unique) - no data Loaded no data for sw_normal_java
Loaded 1 record(s) for sw_flush_java (1x1 unique) - no data Loaded no data for sw_flush_java
Loaded 1 record(s) for sw_noncached_java (1x1 unique) - no data Loaded no data for sw_noncached_java
Loaded 1 record(s) for res_normal_java (1x1 unique) - no data Loaded no data for res_normal_java
Loaded 1 record(s) for res_flush_java (1x1 unique) - no data Loaded no data for res_flush_java
Loaded 1 record(s) for res_noncached_java (1x1 unique) - no data Loaded no data for res_noncached_java
Loaded 1 record(s) for complex_normal_java (1x1 unique) - no data Loaded no data for complex_normal_java
Loaded 1 record(s) for complex_flush_java (1x1 unique) - no data Loaded no data for complex_flush_java
Loaded 1 record(s) for complex_noncached_java (1x1 unique) - no data Loaded no data for complex_noncached_java
Loaded no data for mixed_normal_java
Loaded no data for mixed_flush_java
Loaded no data for mixed_noncached_java
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
java_glpk_dats = read_sol_results('java', since = datetime(2015,6,22,0,0,0)) java_glpk_dats = read_sol_results('java', since = datetime(2015,6,22,0,0,0))
glpk_dats = read_sol_results('glpk') glpk_dats = read_sol_results('glpk')
gurobi_dats = read_sol_results('gurobi') gurobi_dats = read_sol_results('gurobi')
``` ```
%% Output %% Output
Loaded 23 record(s) for update_java (1x23 unique) ~= 1 run(s) Loaded 23 record(s) for update_java (1x23 unique) ~= 1 run(s)
Loaded 1 record(s) for sw_java (1x1 unique) - no data Loaded no data for sw_java
Loaded 1 record(s) for res_java (1x1 unique) - no data Loaded no data for res_java
Loaded 1 record(s) for complex_java (1x1 unique) - no data Loaded no data for complex_java
Loaded no data for mixed_java
Loaded 1656 record(s) for update_glpk (8x23 unique) ~= 9 run(s) Loaded 1656 record(s) for update_glpk (8x23 unique) ~= 9 run(s)
Loaded 1 record(s) for sw_glpk (1x1 unique) - no data Loaded no data for sw_glpk
Loaded 1 record(s) for res_glpk (1x1 unique) - no data Loaded no data for res_glpk
Loaded 1 record(s) for complex_glpk (1x1 unique) - no data Loaded no data for complex_glpk
Loaded no data for mixed_glpk
Loaded 216 record(s) for update_gurobi (8x27 unique) ~= 1 run(s) Loaded 216 record(s) for update_gurobi (8x27 unique) ~= 1 run(s)
Loaded 1 record(s) for sw_gurobi (1x1 unique) - no data Loaded no data for sw_gurobi
Loaded 1 record(s) for res_gurobi (1x1 unique) - no data Loaded no data for res_gurobi
Loaded 1 record(s) for complex_gurobi (1x1 unique) - no data Loaded no data for complex_gurobi
Loaded no data for mixed_gurobi
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
p_ax_nr, p_line_def, p_col_def, p_label = 0, 1, 2, 3 p_ax_nr, p_line_def, p_col_def, p_label = 0, 1, 2, 3
p_gen_racket, p_gen_larceny, p_gen_java = 4, 5, 6 p_gen_racket, p_gen_larceny, p_gen_java = 4, 5, 6
p_sol_glpk, p_sol_gurobi, p_sol_java_glpk = 4, 5, 6 p_sol_glpk, p_sol_gurobi, p_sol_java_glpk = 4, 5, 6
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def draw_gen(changeName, strategy, params): def draw_gen(changeName, strategy, params):
try: try:
name = 'gen_{0}_{1}'.format(changeName, strategy) name = 'gen_{0}_{1}'.format(changeName, strategy)
# needed number of axes equals ax_nr+1 now # needed number of axes equals ax_nr+1 now
f, ax_arr = plt.subplots(nrows = ax_nr+1, ncols = 3, sharex=True, sharey=True) f, ax_arr = plt.subplots(nrows = ax_nr+1, ncols = 3, sharex=True, sharey=True)
f.set_size_inches(25.5,3.5*(ax_nr+1)) f.set_size_inches(25.5,3.5*(ax_nr+1))
one_plot = ax_arr.shape[1] == 1 one_plot = ax_arr.shape[1] == 1
f.patch.set_facecolor('none') f.patch.set_facecolor('none')
f.patch.set_alpha(0.0) f.patch.set_alpha(0.0)
lines, labels = [], [] lines, labels = [], []
for p in params: for p in params:
ax_tup = ax_arr if one_plot else ax_arr[p[p_ax_nr]] ax_tup = ax_arr if one_plot else ax_arr[p[p_ax_nr]]
ax_j = ax_tup[0] ax_j = ax_tup[0]
ax_r = ax_tup[1] ax_r = ax_tup[1]
ax_l = ax_tup[2] ax_l = ax_tup[2]
# ax_j.set_ylim([0,50]) # ax_j.set_ylim([0,50])
# ax_r.set_ylim([0,10]) # ax_r.set_ylim([0,10])
# ax_l.set_ylim([0,10]) # ax_l.set_ylim([0,10])
# x_g = np.array(xrange(1,len(p[1])+1)) # start at one, since first gen-time is cut # x_g = np.array(xrange(1,len(p[1])+1)) # start at one, since first gen-time is cut
x_g = np.array(xrange(START_STEP,len(p[p_gen_racket])+START_STEP)) # start at zero x_g = np.array(xrange(START_STEP,len(p[p_gen_racket])+START_STEP)) # start at zero
line_java = ax_j.plot(x_g, p[p_gen_java][0]*np.ones(len(p[p_gen_racket])), ls = p[p_line_def], line_java = ax_j.plot(x_g, p[p_gen_java][0]*np.ones(len(p[p_gen_racket])), ls = p[p_line_def],
c = p[p_col_def], label = p[p_label]) c = p[p_col_def], label = p[p_label])
line_racket = ax_r.plot(x_g, es(p[p_gen_racket], x_g), ls = p[p_line_def], line_racket = ax_r.plot(x_g, es(p[p_gen_racket], x_g), ls = p[p_line_def],
c = p[p_col_def], label = p[p_label]) c = p[p_col_def], label = p[p_label])
line_larceny = ax_l.plot(x_g, es(p[p_gen_larceny], x_g), ls = p[p_line_def], line_larceny = ax_l.plot(x_g, es(p[p_gen_larceny], x_g), ls = p[p_line_def],
c = p[p_col_def], label = p[p_label]) c = p[p_col_def], label = p[p_label])
ax_l.legend(loc='upper left', bbox_to_anchor=(1, 1.02)) ax_l.legend(loc='upper left', bbox_to_anchor=(1, 1.02))
lines.append(line_racket[0]) lines.append(line_racket[0])
labels.append(p[p_label]) labels.append(p[p_label])
for ax in ax_arr if one_plot else ax_arr.flatten(): for ax in ax_arr if one_plot else ax_arr.flatten():
ax.set_ylabel('seconds') ax.set_ylabel('seconds')
ax.patch.set_alpha(1) ax.patch.set_alpha(1)
ax.patch.set_facecolor('white') ax.patch.set_facecolor('white')
first_ax = ax_arr if one_plot else ax_arr[0] first_ax = ax_arr if one_plot else ax_arr[0]
plt.suptitle('ILP Generation Time', fontsize = 16) plt.suptitle('ILP Generation Time', fontsize = 16)
first_ax[0].set_title('Java') first_ax[0].set_title('Java')
first_ax[1].set_title('Racket') first_ax[1].set_title('Racket')
first_ax[2].set_title('Larceny') first_ax[2].set_title('Larceny')
# Fine-tune figure; make subplots close to each other and hide x ticks for # Fine-tune figure; make subplots close to each other and hide x ticks for
# all but bottom plots. # all but bottom plots.
f.subplots_adjust(hspace=0.2) f.subplots_adjust(hspace=0.2)
plt.setp([a.get_xticklabels() for a in f.axes[:-3]], visible=False) plt.setp([a.get_xticklabels() for a in f.axes[:-3]], visible=False)
plt.setp([a.get_yticklabels() for a in f.axes], visible=True) plt.setp([a.get_yticklabels() for a in f.axes], visible=True)
plt.savefig('doc/{}.pdf'.format(name), facecolor=f.get_facecolor(), edgecolor='none') plt.savefig('doc/{}.pdf'.format(name), facecolor=f.get_facecolor(), edgecolor='none')
plt.savefig('{}.png'.format(name), facecolor=f.get_facecolor(), edgecolor='none') plt.savefig('pngs/{}.png'.format(name), facecolor=f.get_facecolor(), edgecolor='none')
except Exception as e: except Exception as e:
print 'Error while drawing gen in {0}-{1}: {2}'.format(change, strategy, e) print 'Error while drawing gen in {0}-{1}: {2}'.format(change, strategy, e)
traceback.print_exc(file=sys.stdout) traceback.print_exc(file=sys.stdout)
finally: finally:
plt.close() plt.close()
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def draw_sol(changeName, params): def draw_sol(changeName, params):
try: try:
name = 'sol_{}'.format(changeName) name = 'sol_{}'.format(changeName)
f, ax_arr = plt.subplots(nrows = ax_nr+1, ncols = 3, sharex=True, sharey=True) f, ax_arr = plt.subplots(nrows = ax_nr+1, ncols = 3, sharex=True, sharey=True)
f.set_size_inches(25.5,3.5*(ax_nr+1)) f.set_size_inches(25.5,3.5*(ax_nr+1))
one_plot = ax_arr.shape[1] == 1 one_plot = ax_arr.shape[1] == 1
f.patch.set_facecolor('none') f.patch.set_facecolor('none')
f.patch.set_alpha(0.0) f.patch.set_alpha(0.0)
lines, labels = [], [] lines, labels = [], []
for p in params: for p in params:
ax_tup = ax_arr if one_plot else ax_arr[p[p_ax_nr]] ax_tup = ax_arr if one_plot else ax_arr[p[p_ax_nr]]
ax_javaglpk = ax_tup[0] ax_javaglpk = ax_tup[0]
ax_glpk = ax_tup[1] ax_glpk = ax_tup[1]
ax_gurobi = ax_tup[2] ax_gurobi = ax_tup[2]
x = np.array(xrange(0,len(p[p_sol_glpk]))) # start at zero x = np.array(xrange(0,len(p[p_sol_glpk]))) # start at zero
line_javaglpk = ax_javaglpk.plot(x, p[p_sol_java_glpk][0]*np.ones(len(p[p_sol_glpk])), ls = p[p_line_def], line_javaglpk = ax_javaglpk.plot(x, p[p_sol_java_glpk][0]*np.ones(len(p[p_sol_glpk])), ls = p[p_line_def],
c = p[p_col_def], label = p[p_label]) c = p[p_col_def], label = p[p_label])
line_glpk = ax_glpk.plot(x, es(p[p_sol_glpk],x), ls = p[p_line_def], line_glpk = ax_glpk.plot(x, es(p[p_sol_glpk],x), ls = p[p_line_def],
c = p[p_col_def], label = p[p_label]) c = p[p_col_def], label = p[p_label])
line_gurobi = ax_gurobi.plot(x, es(p[p_sol_gurobi],x), ls = p[p_line_def], line_gurobi = ax_gurobi.plot(x, es(p[p_sol_gurobi],x), ls = p[p_line_def],
c = p[p_col_def], label = p[p_label]) c = p[p_col_def], label = p[p_label])
ax_gurobi.legend(loc='upper left', bbox_to_anchor=(1, 1.02)) ax_gurobi.legend(loc='upper left', bbox_to_anchor=(1, 1.02))
lines.append(line_glpk[0]) lines.append(line_glpk[0])
labels.append(p[p_label]) labels.append(p[p_label])
for ax in ax_arr if one_plot else ax_arr.flatten(): for ax in ax_arr if one_plot else ax_arr.flatten():
ax.set_ylabel('seconds') ax.set_ylabel('seconds')
ax.patch.set_alpha(1) ax.patch.set_alpha(1)
ax.patch.set_facecolor('white') ax.patch.set_facecolor('white')
first_ax = ax_arr if one_plot else ax_arr[0] first_ax = ax_arr if one_plot else ax_arr[0]
plt.suptitle('ILP Solving Time', fontsize = 16) plt.suptitle('ILP Solving Time', fontsize = 16)
first_ax[0].set_title('GLPK (Java)') first_ax[0].set_title('GLPK (Java)')
first_ax[1].set_title('GLPK (Scheme)') first_ax[1].set_title('GLPK (Scheme)')
first_ax[2].set_title('Gurobi (Scheme)') first_ax[2].set_title('Gurobi (Scheme)')
# Fine-tune figure; make subplots close to each other and hide x ticks for # Fine-tune figure; make subplots close to each other and hide x ticks for
# all but bottom plots. # all but bottom plots.
f.subplots_adjust(hspace=0.2) f.subplots_adjust(hspace=0.2)
plt.setp([a.get_xticklabels() for a in f.axes[:-3]], visible=False) plt.setp([a.get_xticklabels() for a in f.axes[:-3]], visible=False)
plt.setp([a.get_yticklabels() for a in f.axes], visible=True) plt.setp([a.get_yticklabels() for a in f.axes], visible=True)
plt.savefig('doc/{}.pdf'.format(name)) plt.savefig('doc/{}.pdf'.format(name))
plt.savefig('{}.png'.format(name)) plt.savefig('pngs/{}.png'.format(name))
except Exception as e: except Exception as e:
print 'Error while drawing sol in {0}: {1}'.format(changeName, e) print 'Error while drawing sol in {0}: {1}'.format(changeName, e)
traceback.print_exc(file=sys.stdout) traceback.print_exc(file=sys.stdout)
finally: finally:
plt.close() plt.close()
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def draw_comp_sol(changeName, params): def draw_comp_sol(changeName, params):
try: try:
name = 'comp_sol_{}'.format(changeName) name = 'comp_sol_{}'.format(changeName)
indices = [3, 12, 14, 19] indices = [3, 12, 14, 19]
for i in indices: for i in indices:
p = params[i] p = params[i]
x = np.array(xrange(0,len(p[p_sol_glpk]))) # start at zero x = np.array(xrange(0,len(p[p_sol_glpk]))) # start at zero
line_javaglpk = plt.plot(x, p[p_sol_java_glpk][0]*np.ones(len(p[p_sol_glpk])), ls = linestyles[0], line_javaglpk = plt.plot(x, p[p_sol_java_glpk][0]*np.ones(len(p[p_sol_glpk])), ls = linestyles[0],
c = p[p_col_def], label = p[p_label]) c = p[p_col_def], label = p[p_label])
line_glpk = plt.plot(x, es(p[p_sol_glpk],x), ls = p[p_line_def], c = p[p_col_def]) line_glpk = plt.plot(x, es(p[p_sol_glpk],x), ls = p[p_line_def], c = p[p_col_def])
line_gurobi = plt.plot(x, es(p[p_sol_gurobi],x), ls = p[p_line_def], c = p[p_col_def]) line_gurobi = plt.plot(x, es(p[p_sol_gurobi],x), ls = p[p_line_def], c = p[p_col_def])
plt.legend(loc = 'right') plt.legend(loc = 'right')
plt.ylabel('seconds') plt.ylabel('seconds')
plt.suptitle('ILP Solving Time - Comparison', fontsize = 16) plt.suptitle('ILP Solving Time - Comparison', fontsize = 16)
plt.savefig('doc/{}.pdf'.format(name)) plt.savefig('doc/{}.pdf'.format(name))
plt.savefig('{}.png'.format(name)) plt.savefig('pngs/{}.png'.format(name))
except Exception as e: except Exception as e:
print 'Error while drawing comp-sol in {0}: {1}'.format(changeName, e) print 'Error while drawing comp-sol in {0}: {1}'.format(changeName, e)
traceback.print_exc(file=sys.stdout) traceback.print_exc(file=sys.stdout)
finally: finally:
plt.close() plt.close()
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
## Currently disabled due to fragile and unused output ## Currently disabled due to fragile and unused output
if False: if False:
START_STEP, MAX_PLOTS_IN_ONE = 0, 7 START_STEP, MAX_PLOTS_IN_ONE = 0, 7
for change in kinds['changes']: for change in kinds['changes']:
print 'Change = {}'.format(change) print 'Change = {}'.format(change)
for strategy in kinds['strategies']: for strategy in kinds['strategies']:
ax_nr = 0 ax_nr = 0
print 'Stategy = {}'.format(strategy) print 'Stategy = {}'.format(strategy)
current_plot, ax_nr, last_res = 0, 0, -1 current_plot, ax_nr, last_res = 0, 0, -1
gen_params = [] gen_params = []
for i in xrange(len(specs)): for i in xrange(len(specs)):
current_res = specs[i][2] current_res = specs[i][2]
current_plot += 1 current_plot += 1
if current_plot > MAX_PLOTS_IN_ONE and last_res != current_res: if current_plot > MAX_PLOTS_IN_ONE and last_res != current_res:
ax_nr += 1 ax_nr += 1
current_plot = 0 current_plot = 0
gen_params.append([ax_nr, line_def[i], color_def[i], '{2:d} x ({4}*{5}*{6})'.format(*specs[i]), gen_params.append([ax_nr, line_def[i], color_def[i], '{2:d} x ({4}*{5}*{6})'.format(*specs[i]),
safe(racket_dats[change][strategy],i,START_STEP), safe(racket_dats[change][strategy],i,START_STEP),
safe(larceny_dats[change][strategy],i,START_STEP), safe(larceny_dats[change][strategy],i,START_STEP),
safe(java_dats[change][strategy],i)]) safe(java_dats[change][strategy],i)])
last_res = current_res last_res = current_res
draw_gen(change, strategy, gen_params) draw_gen(change, strategy, gen_params)
# end of for strategies # end of for strategies
sol_params = [] sol_params = []
ax_nr = 0 ax_nr = 0
for i in xrange(len(specs)): for i in xrange(len(specs)):
current_res = specs[i][2] current_res = specs[i][2]
current_plot += 1 current_plot += 1
if current_plot > MAX_PLOTS_IN_ONE and last_res != current_res: if current_plot > MAX_PLOTS_IN_ONE and last_res != current_res:
ax_nr += 1 ax_nr += 1
current_plot = 0 current_plot = 0
sol_params.append([ax_nr, line_def[i], color_def[i], '{2:d} x ({4}*{5}*{6})'.format(*specs[i]), sol_params.append([ax_nr, line_def[i], color_def[i], '{2:d} x ({4}*{5}*{6})'.format(*specs[i]),
safe(glpk_dats[change],i), safe(glpk_dats[change],i),
safe(gurobi_dats[change],i), safe(gurobi_dats[change],i),
safe(java_glpk_dats[change],i)]) safe(java_glpk_dats[change],i)])
draw_sol(change, sol_params) draw_sol(change, sol_params)
draw_comp_sol(change, sol_params) draw_comp_sol(change, sol_params)
``` ```
%% Cell type:code id: tags:
``` python
llong, lshort, lveryshort = 0, 1, 2
l = ['Flushed', '$fs$', 'f']
stt = {'normal' : ['Incremental', '$inc$', 'i'],
'flush' : l, 'flushed': l,
'noncached': ['Noncached', '$nc$', 'n']}
def strategy_to_titel(strategy, length = llong):
try:
return stt[strategy.lower()][length]
except KeyError:
raise Exception('Unknown strategy/length: {0} {1}'.format(strategy, length))
def strategy2_to_titel(ratio):
s1, s2 = ratio.split('To')
return r'${0} \rightarrow {1}$'.format(strategy_to_titel(s1, lshort)[1:-1], strategy_to_titel(s2, lshort)[1:-1])
def change_to_title(change):
if change == 'complex':
change = 'mixed'
return change.title()
```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Boxplots for generation times ## Boxplots for generation times
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
for (data, name) in ((java_dats, 'java'), (racket_dats, 'racket'), (larceny_dats, 'larceny')): for (data, name) in ((java_dats, 'java'), (racket_dats, 'racket'), (larceny_dats, 'larceny')):
print name+':' print name+':'
#change, strategy = 'update', 'normal' #change, strategy = 'update', 'normal'
for change in kinds['changes']: for change in kinds['changes']:
#if True: #if True:
sys.stdout.write(change+' ') sys.stdout.write(change+' ')
# if True: # if True:
for strategy in kinds['strategies']: for strategy in kinds['strategies']:
sys.stdout.write(strategy+',') sys.stdout.write(strategy+',')
cdat = data[change][strategy] cdat = data[change][strategy]
plt.boxplot(cdat.transpose()) plt.boxplot(cdat.transpose())
axes = plt.gca() axes = plt.gca()
axes.set_ylim([0,50]) axes.set_ylim([0,50])
plt.title('{0} {1} {2}'.format(name.title(), change.title(), strategy.title())) plt.title('{0} {1} {2}'.format(name.title(), change_to_title(change), strategy_to_titel(strategy)))
plt.ylabel('seconds') plt.ylabel('seconds')
plt.xlabel('time') plt.xlabel('step')
plt.savefig('gen_bp_{0}_{1}_{2}.png'.format(name,change,strategy)) plt.savefig('pngs/gen_bp_{0}_{1}_{2}.png'.format(name,change,strategy))
plt.savefig('doc/gen_bp_{0}_{1}_{2}.pdf'.format(name,change,strategy)) plt.savefig('doc/gen_bp_{0}_{1}_{2}.pdf'.format(name,change,strategy))
plt.close() plt.close()
print '' print ''
``` ```
%% Output %% Output
java: java:
update normal,flush,noncached,sw normal,flush,noncached,res normal,flush,noncached,complex normal,flush,noncached, update normal,flush,noncached,sw normal,flush,noncached,res normal,flush,noncached,complex normal,flush,noncached,mixed normal,flush,noncached,
racket: racket:
update normal,flush,noncached,sw normal,flush,noncached,res normal,flush,noncached,complex normal,flush,noncached, update normal,flush,noncached,sw normal,flush,noncached,res normal,flush,noncached,complex normal,flush,noncached,mixed normal,flush,noncached,
larceny: larceny:
update normal,flush,noncached,sw normal,flush,noncached,res normal,flush,noncached,complex normal,flush,noncached, update normal,flush,noncached,sw normal,flush,noncached,res normal,flush,noncached,complex normal,flush,noncached,mixed normal,flush,noncached,
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Boxplot for solving times ## Boxplot for solving times
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
for (data, name) in ((java_glpk_dats, 'java'), (glpk_dats, 'glpk'), (gurobi_dats, 'gurobi')): for (data, name) in ((java_glpk_dats, 'java'), (glpk_dats, 'glpk'), (gurobi_dats, 'gurobi')):
print name+':' print name+':'
#change = 'update' #change = 'update'
for change in kinds['changes']: for change in kinds['changes']:
#if True: #if True:
sys.stdout.write(change+' ') sys.stdout.write(change+' ')
# if True: # if True:
cdat = data[change] cdat = data[change]
plt.boxplot(cdat.transpose()) plt.boxplot(cdat.transpose())
plt.title('{0} {1}'.format(name.title(), change.title())) plt.title('{0} {1}'.format(name.title(), change_to_title(change)))
plt.ylabel('seconds') plt.ylabel('seconds')
plt.xlabel('time') plt.xlabel('step')
plt.savefig('sol_bp_{0}_{1}.png'.format(name,change)) plt.savefig('pngs/sol_bp_{0}_{1}.png'.format(name,change))
plt.savefig('doc/sol_bp_{0}_{1}_{2}.pdf'.format(name,change,strategy)) plt.savefig('doc/sol_bp_{0}_{1}.pdf'.format(name,change))
plt.close() plt.close()
print '' print ''
``` ```
%% Output %% Output
java: java:
update sw res complex update sw res complex mixed
glpk: glpk:
update sw res complex update sw res complex mixed
gurobi: gurobi:
update sw res complex update sw res complex mixed
%% Cell type:markdown id: tags:
## Times for initial generation of the ILP
%% Cell type:code id: tags:
``` python
print reduce(lambda x,y: np.concatenate((x,y)),[racket_dats[change][strategy][0] for strategy in kinds['strategies']
for change in kinds['changes']]).astype(int), '<- Racket'
print reduce(lambda x,y: np.concatenate((x,y)),[larceny_dats[change][strategy][0] for strategy in kinds['strategies']
for change in kinds['changes']]).astype(int), '<- Larceny'
```
%% Output
[ 0 0 2 15 0 0 3 21 1 1 2 5 9 10 13 13 17 22 27 20 25 30 37 2 6
10 14 0 0 1 7 0 0 1 14 0 0 3 19 1 1 2 5 8 10 12 14 17 21 25
19 24 28 34 3 6 10 14 0 0 1 6 0 0 1 14 0 0 3 19 1 1 2 5 8
10 12 14 17 21 25 19 23 29 35 2 6 10 14 0 0 1 6 0 5 9 11 13 14 22
2 7 11 15 0 0 1 15 0 0 3 22 1 1 2 5 9 10 13 14 18 22 28 20 26
30 36 2 6 10 14 0 0 1 8 0 0 1 15 0 0 2 17 1 1 2 4 7 9 10
11 14 18 24 16 20 25 30 2 5 9 12 0 0 1 6 0 0 1 15 0 0 2 17 1
1 2 4 8 9 11 11 14 18 22 16 20 25 31 2 5 8 12 0 0 1 6 0 0 0
2 17 0 0 4 27 1 3 4 8 12 15 17 20 26 30 36 28 34 40 47 4 10 17 23
0 1 2 8 0 0 2 15 0 0 4 27 1 3 4 8 12 15 17 20 24 30 36 28 33
40 47 4 10 16 22 0 1 2 8 0 0 2 15 0 0 4 26 1 3 4 8 12 14 17
19 24 30 35 28 33 40 48 5 10 16 24 0 1 2 9 0] <- Racket
[ 0 0 1 11 0 0 3 22 1 2 3 6 10 12 15 18 22 27 32 25 30 38 45 4 9
16 24 0 0 1 3 0 0 0 0 0 0 0 0 0 0 0] <- Larceny
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Attribute metrics ## Attribute metrics
- X = {normal, flushed, noncached} - X = {normal, flushed, noncached}
- (X1,X2) = {(normal, flushed), (normal, noncached), (flushed, noncached)} - (X1,X2) = {(normal, flushed), (normal, noncached), (flushed, noncached)}
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def read_att_result(name = 'profiling/att-percentages.csv', adjust_number = True): def read_att_result(name = 'profiling/att-percentages.csv', adjust_number = True):
def cdir(text): def cdir(text):
prefix = text[:-4] prefix = text[:-4]
number = int(text[-3:]) number = int(text[-3:])
if not adjust_number or prefix == 'update': if not adjust_number or prefix == 'update':
return number return number
elif prefix == 'sw': elif prefix == 'sw':
return 31+number return 31+number
elif prefix == 'res': elif prefix == 'res':
return 62+number return 62+number
elif prefix == 'complex': elif prefix == 'complex':
return 93+number return 93+number
elif prefix == 'mixed':
return 104+number
elif prefix == '': elif prefix == '':
pass pass
else: else:
print 'Unknown prefix "{0}"'.format(prefix) print 'Unknown prefix "{0}"'.format(prefix)
dat = np.genfromtxt(name, delimiter=',', names=True, dat = np.genfromtxt(name, delimiter=',', names=True,
#dtype=(int, float, float, float, float, float, float, float), #dtype=(int, float, float, float, float, float, float, float),
converters={'dir': cdir}) converters={'dir': cdir})
dat.sort(axis=0) dat.sort(axis=0)
# check for incomplete values print 'Loaded {0} ({1} zero-values) attribute metric run(s) from {2}'.format(dat.size,
reduce(lambda total,row: total + 1*(row == 0.0) if isinstance(row,int) or isinstance(row,float) else
print 'Loaded {0} ({1} zero-values) attribute metric run(s) from {2}'.format( reduce(lambda x,y: x+1 if y == 0 else x, row, 0),dat.tolist(),0), name)
len(dat), reduce(lambda total,row: total + reduce(lambda x,y: x+1 if y == 0 else x, row, 0),dat,0), name)
return dat return dat
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
att_dat = read_att_result() att_dat = read_att_result()
att_dat.dtype att_dat.dtype
``` ```
%% Output %% Output
Loaded 104 (173 zero-values) attribute metric run(s) from profiling/att-percentages.csv Loaded 105 (8 zero-values) attribute metric run(s) from profiling/att-percentages.csv
dtype([('dir', '<f8'), ('normalBaseline', '<f8'), ('flushedBaseline', '<f8'), ('noncachedBaseline', '<f8'), ('ratioNormalToFlushed', '<f8'), ('ratioNormalToNoncached', '<f8'), ('ratioFlushedToNoncached', '<f8'), ('speedupNormalToFlushed', '<f8'), ('speedupNormalToNoncached', '<f8'), ('speedupFlushedToNoncached', '<f8')]) dtype([('dir', '<f8'), ('normalBaseline', '<f8'), ('flushedBaseline', '<f8'), ('noncachedBaseline', '<f8'), ('ratioNormalToFlushed', '<f8'), ('ratioNormalToNoncached', '<f8'), ('ratioFlushedToNoncached', '<f8'), ('speedupNormalToFlushed', '<f8'), ('speedupNormalToNoncached', '<f8'), ('speedupFlushedToNoncached', '<f8')])
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
1) {X}basline: `total.X.computed / total.X.called` 1) {X}basline: `total.X.computed / total.X.called`
- the baseline of the method X - the baseline of the method X
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
width = 0.5 width = 0.5
x = att_dat['dir'] x = att_dat['dir']
for n, (l, c) in enumerate((('noncached', 'grey'), ('flushed', 'g'), ('normal', 'b'))): for n, (l, c) in enumerate((('noncached', 'grey'), ('flushed', 'g'), ('normal', 'b'))):
y = att_dat[l + 'Baseline'] y = att_dat[l + 'Baseline']
#plt.plot(x, y, label = l, color = c) #plt.plot(x, y, label = l, color = c)
plt.bar(x*4*width + n*width, y, width, label = l, color = c) plt.bar(x*4*width + n*width, y, width, label = strategy_to_titel(l), color = c)
plt.legend(loc = 'best') plt.legend(loc = 'best')
plt.ylabel('% computed') plt.ylabel('% computed')
plt.suptitle('Attribute Baselines', fontsize = 16) plt.suptitle('Attribute Baselines', fontsize = 16)
plt.savefig('doc/att_bl.pdf') plt.savefig('doc/att_bl.pdf')
!pdfcrop doc/att_bl.pdf doc/att_bl_cropped.pdf > /dev/null !pdfcrop doc/att_bl.pdf doc/att_bl_cropped.pdf > /dev/null
plt.savefig('att_bl.png') plt.savefig('pngs/att_bl.png')
plt.close() plt.close()
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
2) ratio{X1}To{X2}: `total.{X1}.computed / total.{X2}.called` 2) ratio{X1}To{X2}: `total.{X1}.computed / total.{X2}.called`
- the efficiency of the incremental approach in comparison to the method X2, - the efficiency of the incremental approach in comparison to the method X2,
i.e. the ratio between actual work done in X1 compared to possible work done with method X2 i.e. the ratio between actual work done in X1 compared to possible work done with method X2
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
x = att_dat['dir'] x = att_dat['dir']
for n, (name, c, ls) in enumerate((('FlushedToNoncached', 'orange', '--'), for n, (name, c, ls) in enumerate((('FlushedToNoncached', 'orange', '--'),
('NormalToFlushed', 'purple', '-'), ('NormalToFlushed', 'purple', '-'),
('NormalToNoncached', 'r', '-.'))): ('NormalToNoncached', 'r', '-.'))):
y = att_dat['ratio' + name] y = att_dat['ratio' + name]
plt.plot(x, y, label = name.replace('To', r'$\rightarrow$'), c = c, ls = ls) plt.plot(x, y, label = strategy2_to_titel(name), c = c, ls = ls)
#plt.bar(x*3*width + n*width, y, width, label = l, color = c) #plt.bar(x*3*width + n*width, y, width, label = l, color = c)
plt.legend(loc = 'best') plt.legend(loc = 'best')
plt.ylabel('%') plt.ylabel('%')
plt.suptitle('Attribute Ratios', fontsize = 16) plt.suptitle('Attribute Ratios', fontsize = 16)
plt.savefig('doc/att_r.pdf') plt.savefig('doc/att_r.pdf')
!pdfcrop doc/att_r.pdf doc/att_r_cropped.pdf > /dev/null !pdfcrop doc/att_r.pdf doc/att_r_cropped.pdf > /dev/null
plt.savefig('att_r.png') plt.savefig('pngs/att_r.png')
plt.close() plt.close()
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
3) speedup{X1}To{X2}: `(total.{X2}.computed / total.{X2}.called) - (total.{X1}.computed / total.{X2}.called)` 3) speedup{X1}To{X2}: `(total.{X2}.computed / total.{X2}.called) - (total.{X1}.computed / total.{X2}.called)`
- = `baseline({X2}) - ratio({X1}, {X2})` - = `baseline({X2}) - ratio({X1}, {X2})`
- the "speed-up" of the incremental approach (normal or flushed) in comparison to the method X2 - the "speed-up" of the incremental approach (normal or flushed) in comparison to the method X2
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
x = att_dat['dir'] x = att_dat['dir']
for n, (l, c) in enumerate((('NormalToNoncached', 'r'), for n, (l, c) in enumerate((('NormalToNoncached', 'r'),
('FlushedToNoncached', 'orange'), ('FlushedToNoncached', 'orange'),
('NormalToFlushed', 'yellow'))): ('NormalToFlushed', 'yellow'))):
y = att_dat['speedup' + l] y = att_dat['speedup' + l]
plt.plot(x, y, label = l.replace('To', r'$\rightarrow$'), color = c) plt.plot(x, y, label = strategy2_to_titel(l), color = c)
#plt.bar(x*3*width + n*width, y, width, label = l, color = c) #plt.bar(x*3*width + n*width, y, width, label = l, color = c)
plt.legend(loc = 'best') plt.legend(loc = 'best')
plt.ylabel('%') plt.ylabel('%')
plt.suptitle('Attribute Speed-Ups', fontsize = 16) plt.suptitle('Attribute Speed-Ups', fontsize = 16)
plt.savefig('doc/att_sp.pdf') plt.savefig('doc/att_sp.pdf')
!pdfcrop doc/att_sp.pdf doc/att_sp_cropped.pdf > /dev/null !pdfcrop doc/att_sp.pdf doc/att_sp_cropped.pdf > /dev/null
plt.savefig('att_sp.png') plt.savefig('pngs/att_sp.png')
plt.close() plt.close()
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Boxplots for attribute measures ## Boxplots for attribute measures
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
att_totals = {} att_totals = {change: read_att_result(name = 'profiling/splitted/att-percentages_{}.csv'.format(change),
for change in kinds['changes']: adjust_number = False) for change in kinds['changes']}
att_totals[change] = read_att_result(name = 'profiling/splitted/att-percentages_{}.csv'.format(change),
adjust_number = False)
``` ```
%% Output %% Output
Loaded 31 (0 zero-values) attribute metric run(s) from profiling/splitted/att-percentages_update.csv Loaded 31 (0 zero-values) attribute metric run(s) from profiling/splitted/att-percentages_update.csv
Loaded 31 (0 zero-values) attribute metric run(s) from profiling/splitted/att-percentages_sw.csv Loaded 31 (0 zero-values) attribute metric run(s) from profiling/splitted/att-percentages_sw.csv
Loaded 31 (85 zero-values) attribute metric run(s) from profiling/splitted/att-percentages_res.csv Loaded 31 (0 zero-values) attribute metric run(s) from profiling/splitted/att-percentages_res.csv
Loaded 11 (88 zero-values) attribute metric run(s) from profiling/splitted/att-percentages_complex.csv Loaded 11 (0 zero-values) attribute metric run(s) from profiling/splitted/att-percentages_complex.csv
Loaded 1 (8 zero-values) attribute metric run(s) from profiling/splitted/att-percentages_mixed.csv
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
PRINT_NONCACHED = True PRINT_NONCACHED = True
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def ratio_columns(name): def draw_att(x, fcolumn, flabel, title, fname_part, rotation_xaxis = None):
if name == 'normal':
return 'ratioNormalToFlushed'
if name == 'flushed':
return 'ratioNormalToNoncached'
if name == 'noncached':
return 'ratioFlushedToNoncached'
raise Exception('Unknown name: {}'.format(name))
def ratio_speedup_names(name):
if name == 'normal':
return r'nm$\rightarrow$fs'
if name == 'flushed':
return r'nm$\rightarrow$nc'
if name == 'noncached':
return r'fs$\rightarrow$nc'
raise Exception('Unknown name: {}'.format(name))
def speedup_columns(name):
if name == 'normal':
return 'speedupNormalToFlushed'
if name == 'flushed':
return 'speedupNormalToNoncached'
if name == 'noncached':
return 'speedupFlushedToNoncached'
raise Exception('Unknown name: {}'.format(name))
```
%% Cell type:code id: tags:
``` python
def draw_att(fcolumn, flabel, title, fname_part, rotation_xaxis = None):
""" Draw boxplots for attribute """ Draw boxplots for attribute
@param:fcolum: function with one parameter (name) returning column name @param:fcolum: function with one parameter (name) returning column name
@param:flabel: function with one parameter (name) returning label @param:flabel: function with one parameter (name) returning label
@param:title: title of plot @param:title: title of plot
@param:fname_part: unique filename part for storing plot @param:fname_part: unique filename part for storing plot
@param:rotation_xaxis: rotation of xaxis ticks (default: no rotation)""" @param:rotation_xaxis: rotation of xaxis ticks (default: no rotation)"""
try: try:
fig, axes = plt.subplots(ncols=len(kinds['changes'])) fig, axes = plt.subplots(ncols=len(kinds['changes']))
# if not rotation_xaxis: # if not rotation_xaxis:
# print dir(fig) # print dir(fig)
fig.suptitle(title, fontsize = 16) fig.suptitle(title, fontsize = 16)
l = ['normal', 'flushed'] l = ['normal', 'flushed']
if PRINT_NONCACHED: if PRINT_NONCACHED:
l.append('noncached') l.append('noncached')
for i, change in enumerate(kinds['changes']): for i, change in enumerate(kinds['changes']):
att_dat = att_totals[change] att_dat = att_totals[change]
axes[i].boxplot([att_dat[fcolumn(name)] for name in l], labels = [flabel(name) for name in l]) axes[i].boxplot([att_dat[fcolumn(name)] for name in x],
axes[i].set_title(change, fontsize=10) labels = [flabel(name) for name in x])
axes[i].set_title(change_to_title(change), fontsize=10)
if i > 0: if i > 0:
axes[i].set_yticklabels([]) axes[i].set_yticklabels([])
for ax in axes.flatten(): for ax in axes.flatten():
if PRINT_NONCACHED: if PRINT_NONCACHED:
ax.set_ylim([0,1.05]) ax.set_ylim([0,1.05])
if rotation_xaxis: if rotation_xaxis:
plt.setp(ax.xaxis.get_majorticklabels(), rotation=rotation_xaxis) plt.setp(ax.xaxis.get_majorticklabels(), rotation=rotation_xaxis)
# plt.axes().xaxis.set_major_locator(matplotlib.ticker.FixedLocator(range(0, 3, 2))) # plt.axes().xaxis.set_major_locator(matplotlib.ticker.FixedLocator(range(0, 3, 2)))
# plt.axes().xaxis.set_minor_locator(matplotlib.ticker.FixedLocator([1])) # plt.axes().xaxis.set_minor_locator(matplotlib.ticker.FixedLocator([1]))
# plt.axes().xaxis.set_minor_formatter(matplotlib.ticker.FormatStrFormatter("%s")) # plt.axes().xaxis.set_minor_formatter(matplotlib.ticker.FormatStrFormatter("%s"))
# plt.axes().tick_params(which='major', pad=20, axis='x') # plt.axes().tick_params(which='major', pad=20, axis='x')
plt.subplots_adjust(top=0.85) plt.subplots_adjust(top=0.85)
plt.savefig('doc/{}.pdf'.format(fname_part)) plt.savefig('doc/{}.pdf'.format(fname_part))
!pdfcrop {'doc/{}.pdf'.format(fname_part)} {'doc/{}_cropped.pdf'.format(fname_part)} > /dev/null !pdfcrop {'doc/{}.pdf'.format(fname_part)} {'doc/{}_cropped.pdf'.format(fname_part)} > /dev/null
plt.savefig('{}.png'.format(fname_part)) plt.savefig('pngs/{}.png'.format(fname_part))
except Exception as e: except Exception as e:
print 'Error while drawing att in {0}: {1}'.format(change, e) print 'Error while drawing att in {0}: {1}'.format(change, e)
traceback.print_exc(file=sys.stdout) traceback.print_exc(file=sys.stdout)
finally: finally:
plt.close() plt.close()
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
draw_att(lambda x: '{}Baseline'.format(x), lambda x: x[0]+x[3], 'Attribute Baselines', 'att_box_bl') def baseline(x):
draw_att(ratio_columns, ratio_speedup_names, 'Attribute Ratios', 'att_box_r', rotation_xaxis = 30) return '{}Baseline'.format('flushed' if x == 'flush' else x)
draw_att(speedup_columns, ratio_speedup_names, 'Attribute Speedups', 'att_box_sp', rotation_xaxis = 30) draw_att(kinds['strategies'], baseline, functools.partial(strategy_to_titel, length = lshort),
'Attribute Baselines', 'att_box_bl')
x = ['NormalToFlushed', 'NormalToNoncached', 'FlushedToNoncached']
draw_att(x, lambda x: 'ratio{}'.format(x), strategy2_to_titel, 'Attribute Ratios', 'att_box_r', rotation_xaxis = 30)
draw_att(x, lambda x: 'speedup{}'.format(x), strategy2_to_titel, 'Attribute Speedups', 'att_box_sp', rotation_xaxis = 30)
``` ```
... ...
......
ilp-measurement.scm
+ 63
− 11
View file @ 100d06e2
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
(library (library
(mquat ilp-measurement) (mquat ilp-measurement)
(export measurement-cli-call) (export measurement-cli-call)
(import (rnrs) (racr core) (srfi :19) (only (srfi :13) string-prefix? string-suffix?) (import (rnrs) (racr core) (srfi :19) (only (srfi :13) string-prefix? string-suffix? string-pad)
(mquat ast) (mquat basic-ag) (mquat utils) (mquat join) (mquat ilp) (mquat constants) (mquat ast) (mquat basic-ag) (mquat utils) (mquat join) (mquat ilp) (mquat constants)
(mquat ast-generation) (mquat properties)) (mquat ast-generation) (mquat properties))
...@@ -16,6 +16,12 @@ ...@@ -16,6 +16,12 @@
(define sw-test 'sw-test) (define sw-test 'sw-test)
(define update-test 'update-test) (define update-test 'update-test)
(define complex-test 'complex-test) (define complex-test 'complex-test)
(define mixed-test 'mixed-test)
(define raw-mixed-params
; HW = number hw. S = number of sub-per-hw. #C[omp], #I[mpl], #M[ode]
; ID HW S #C #I #M
(list (list 1 300 0 10 2 2)
(list 2 400 0 10 2 2)))
(define raw-short-params (define raw-short-params
; HW = number hw. S = number of sub-per-hw. #C[omp], #I[mpl], #M[ode] ; HW = number hw. S = number of sub-per-hw. #C[omp], #I[mpl], #M[ode]
; ID HW S #C #I #M ; ID HW S #C #I #M
...@@ -79,10 +85,10 @@ ...@@ -79,10 +85,10 @@
(map (lambda (l) (append (cons* (dirname "complex-" (car l)) complex-test (cdr l)) (map (lambda (l) (append (cons* (dirname "complex-" (car l)) complex-test (cdr l))
(list (list (lambda _ #t) #f #f #f)))) (list (list (lambda _ #t) #f #f #f))))
raw-short-params) raw-short-params)
; ; normal params ; mixed params
; (map (lambda (l) (append (cons* (dirname "update-" (car l)) update-test (cdr l)) (map (lambda (l) (append (cons* (dirname "mixed-" (car l)) mixed-test (cdr l))
; (list (list (lambda _ #t) #f #f #f)))) (list (list (lambda _ #t) #f #f #f))))
; raw-params) raw-mixed-params)
)) ))
(define (valf val) (lambda _ val)) (define (valf val) (lambda _ val))
...@@ -179,7 +185,8 @@ ...@@ -179,7 +185,8 @@
(define (find-create ast l prefix lon make-new) (define (find-create ast l prefix lon make-new)
(let ([name (node-name prefix lon)]) (let ([name (node-name prefix lon)])
(or (ast-find-child (lambda (i child) (string=? (->name child) name)) l) (make-new)))) (or (ast-find-child (lambda (i child) (string=? (->name child) name)) l) (make-new))))
(define (find-create-comp ast comp-nr) (find-create ast (->Comp* (->SWRoot ast)) "c" (list comp-nr) (lambda _ (add-comp ast comp-nr)))) (define (find-create-comp ast comp-nr) (find-create ast (->Comp* (->SWRoot ast)) "c" (list comp-nr)
(lambda _ (add-comp ast comp-nr))))
(define (add-impl ast comp-nr impl-nr reqcomps) (define (add-impl ast comp-nr impl-nr reqcomps)
(debug "#create new impl" comp-nr impl-nr reqcomps) (debug "#create new impl" comp-nr impl-nr reqcomps)
(let ([new (:Impl mquat-spec (node-name "i" (list impl-nr comp-nr)) (list) (let ([new (:Impl mquat-spec (node-name "i" (list impl-nr comp-nr)) (list)
...@@ -289,17 +296,62 @@ ...@@ -289,17 +296,62 @@
(rw* rt "load" #f ast) (sit id-s "10-add-odd-pes" ast) (display+flush ".")) (rw* rt "load" #f ast) (sit id-s "10-add-odd-pes" ast) (display+flush "."))
(delete-odd-modes ast) (sit id-s "11-del-modes" ast) (display+flush "."))) (delete-odd-modes ast) (sit id-s "11-del-modes" ast) (display+flush ".")))
; make change kind frequency according to some measure, e.g. 80% update, 10% res, 5% sw, 5% no change
; = 5x (16 update, 2 res, 1 sw, 1 no)
; = 5x (3 update + res + 3 update + sw + 4 update + no + 3 update + res + 3 update)
(define (run-mixed-test id-s specs)
(define ast (cst id-s specs))
(define rt (ast-child 1 (->ResourceType* (->HWRoot ast))))
(define pe+parent (lambda (pe) (cons pe (<- pe))))
(define odd-pes (map pe+parent (ret-odds (=every-container ast))))
(define (make-step-name outer step suffix) (string-append (string-pad (number->string (+ (* 20 outer) step)) 2 #\0) suffix))
(define (add-new-resource)
(let* ([max-id (apply max (map (lambda (pe) (string->number (substring (->name pe) 2))) (=every-container ast)))]
[first-clauses (->* (->ProvClause* (car (=every-container ast))))]
[new-clauses (map (lambda (cl) (make-prov (->return-type cl) (->comparator cl) (rand 50 2 0))) first-clauses)] ;TODO
[new-res (:Resource mquat-spec (number->string (+ 1 max-id)) rt online (list) new-clauses)])
(rewrite-add (->SubResources (->HWRoot ast)) new-res)))
(define (update-change outer steps)
(for-each
(lambda (step)
(when (not (and (= outer 4) (= step 20)))
(rw* rt "load" #f ast) (sit id-s (make-step-name outer step "-every-comp-rand") ast) (display+flush "."))) steps))
(rewrite-terminal 'config ast id-s)
(display+flush id-s)
; (info (map (lambda (pe) (string->number (substring (->name pe) 2))) (=every-container ast)))
(add-new-resource) ;; <- just for testing
(sit id-s "00-init" ast)
(for-each
(lambda (outer)
(update-change outer (list 1 2 3))
; (let ([removed (map (lambda (pp) (cons (rewrite-delete (car pp)) (cdr pp))) odd-pes)])
; (sit id-s (make-step-name outer 4 "-del-odd-pes") ast) (display+flush ".")
(add-new-resource) (sit id-s (make-step-name outer 4 "-add-new-pes") ast) (display+flush ".")
(update-change outer (list 5 6 7))
(make-new-modes ast) (sit id-s (make-step-name outer 8 "-new-modes") ast) (display+flush ".")
(update-change outer (list 9 10 11 12))
(sit id-s (make-step-name outer 13 "-no-change") ast)
(update-change outer (list 14 15 16))
; (for-each (lambda (rp) (rewrite-add (cdr rp) (car rp))) (reverse removed))
; (rw* rt "load" #f ast) (sit id-s (make-step-name outer 17 "-add-odd-pes") ast) (display+flush "."))
(add-new-resource) (sit id-s (make-step-name outer 17 "-add-new-pes") ast) (display+flush ".")
(update-change outer (list 18 19 20))) (list 0 1 2 3 4)))
(define tests (list (cons resource-test run-resource-test) (define tests (list (cons resource-test run-resource-test)
(cons sw-test run-sw-test) (cons sw-test run-sw-test)
(cons update-test run-update-test) (cons update-test run-update-test)
(cons complex-test run-complex-test))) (cons complex-test run-complex-test)
(cons mixed-test run-mixed-test)))
(define (run-test id-s specs) (define (run-test id-s specs)
(when profiling? (reset-counts)) (when profiling? (reset-counts))
(let ([f (assoc (car specs) tests)]) (let ([f (assoc (car specs) tests)])
(if f ((cdr f) id-s (cdr specs)) (error "Unknown kind" (car specs))))) (if f ((cdr f) id-s (cdr specs)) (error "Unknown kind" (car specs)))))
(define (print-usage) (error "measurement-cli-call" "No valid arguments found, use 'all', 'dirs', 'prefix', 'suffix' or a number of ids.")) (define (print-usage) (error "measurement-cli-call"
"No valid arguments found, use 'all', 'dirs', 'prefix', 'suffix' or a number of ids."))
(define (measurement-cli-call command-line) (define (measurement-cli-call command-line)
(let ([first (if (= 0 (length command-line)) #f (car command-line))]) (let ([first (if (= 0 (length command-line)) #f (car command-line))])
... ...
......
ilp_measurement.py
+ 5
− 4
View file @ 100d06e2
...@@ -29,12 +29,14 @@ all_results = 'all.csv' ...@@ -29,12 +29,14 @@ all_results = 'all.csv'
class timed(object): class timed(object):
def __enter__(self, msg = ' done in {0:.3f}s'): def __enter__(self, msg = ' done in {0:.3f}s'):
print 'Starting measurement at {:%d, %b %Y at %H:%M:%S.%f}'.format(datetime.today())
self.start = timeit.default_timer() self.start = timeit.default_timer()
self.msg = msg self.msg = msg
return self return self
def __exit__(self, ex_type, value, traceback): def __exit__(self, ex_type, value, traceback):
self.stop = timeit.default_timer() - self.start self.stop = timeit.default_timer() - self.start
print self.msg.format(self.stop) print self.msg.format(self.stop)
print 'Finished measurement at {:%d, %b %Y at %H:%M:%S.%f}'.format(datetime.today())
@task(name = 'current-ids') @task(name = 'current-ids')
def current_ids(): def current_ids():
...@@ -475,14 +477,13 @@ def incoporate_remote_measurements(archive, dryrun = False): ...@@ -475,14 +477,13 @@ def incoporate_remote_measurements(archive, dryrun = False):
sys.stdout.write('\n') sys.stdout.write('\n')
@task @task
def factor(incremental, flushed, noncached): def factor(column, *files):
def get_average_time(f): def get_average_time(f):
with open(f) as fd: with open(f) as fd:
r = csv.reader(fd) r = csv.reader(fd)
values = [float(row[-1]) for row in r if not row[0].isalpha()] values = [float(row[int(column)]) for row in r if not row[0].isalpha()]
return sum(values) / float(len(values)) return sum(values) / float(len(values))
avi, avf, avn = map(get_average_time, [incremental, flushed, noncached]) print map(get_average_time, files)
print avi, avf, avn
if __name__ == '__main__': if __name__ == '__main__':
check() check()
... ...
......
profiling/kinds.json
+ 1
− 1
View file @ 100d06e2
{"strategies": ["normal", "flush", "noncached"], "changes": ["update", "sw", "res", "complex"]} {"strategies": ["normal", "flush", "noncached"], "changes": ["update", "sw", "res", "complex", "mixed"]}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment