Skip to content
Snippets Groups Projects
Commit 4be698eb authored by René Schöne's avatar René Schöne
Browse files

WIP: again towards working graph creation

parent 15bd11a9
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
ilp-measurement.ipynb
+ 150
− 41
View file @ 4be698eb
%% Cell type:code id: tags:
``` python
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.axes
%matplotlib inline
plt.ioff()
```
%% Cell type:code id: tags:
``` python
from glob import glob
from datetime import datetime
from itertools import *
from operator import itemgetter
import json
```
%% Cell type:code id: tags:
``` python
spec_files = glob('profiling/*/specs')
spec_files.sort()
specs = [np.genfromtxt(f, delimiter=' ', dtype=int)
for f in spec_files]
```
%% Cell type:code id: tags:
``` python
linestyles = ["-","--","-.",":"]
colors = ('blue','black','red','orange','green','magenta','yellow', 'cyan', 'purple', 'firebrick')
colorcycle = cycle(colors)
lastpe = False
line_def = []
color_def = []
for spec in specs:
pe, comp, impl, mode = (spec[2], spec[4], spec[5], spec[6])
if not lastpe or lastpe != pe:
color = next(colorcycle)
linecycle = cycle(linestyles)
line = next(linecycle)
lastpe = pe
line_def.append(line)
color_def.append(color)
```
%% Cell type:code id: tags:
``` python
def set_keys(*indices):
"""Returns a function that returns a tuple of key values"""
def get_keys(seq, indices=indices):
keys = []
for i in indices:
keys.append(seq[i])
return tuple(keys)
return get_keys
```
%% Cell type:code id: tags:
``` python
def get_average_times(dat, dirCol, stepCol, timeCol):
if dat.size == 1:
return np.array([np.array([dat.item()[timeCol]])])
dat.sort(order=['dir', 'step'])
result = {}
for (c_dir, c_step), rows in groupby(dat, key=set_keys('dir','step')):
#print c_dir, c_step, rows
c_dir, c_step, total_time, counter = None, None, 0, 0
for row in rows:
if not c_dir:
c_dir = row[dirCol]
if not c_step:
c_step = row[stepCol]
total_time+=row[timeCol]
counter += 1
result.setdefault(c_dir, []).append([c_step, total_time*1.0/counter])
result2 = []
for c_dir, rows in result.iteritems():
inner = []
for row in rows:
inner.append(row[1])
result2.append(inner)
result2.append([row[1] for row in rows])
# for row in rows:
# inner.append(row[1])
# result2.append(inner)
return np.array([np.array(rows) for rows in result2])
```
%% Cell type:code id: tags:
``` python
print 'gen', larceny_dats['update']['normal'].shape
print 'sol', glpk_dats['update'].shape
```
%% Output
gen (8, 31)
sol (8, 23)
%% Cell type:code id: tags:
``` python
def is_axis(a):
return not type(a) is np.ndarray
```
%% Cell type:code id: tags:
``` python
with open('profiling/kinds.json') as fd:
kinds = json.load(fd)
print 'Kinds: {}'.format(kinds)
```
%% Output
Kinds: {u'strategies': [u'normal', u'flush', u'noncached'], u'changes': [u'update', u'sw', u'res']}
%% Cell type:markdown id: tags:
Header = ['timestamp', 'dir', 'step', 'ilp-gen']
%% Cell type:code id: tags:
``` python
no_split, split_change_only, split_both = 0, 1, 2
dat, dat2, dat3 = None, None, None
def read_single_result(f, name, dtype, data_column, since):
global dat, dat2, dat3
def convdate(text):
return datetime.strptime(text, '%Y-%m-%dT%H:%M:%S.%f')
def convdir(text):
return int(text[-3:])
def convstep(text):
return int(text[0:2])
def safe_div(a, b):
return a/b if b>0 else "{}!".format(a)
dat = np.genfromtxt(f, delimiter=',', names=True, dtype=dtype,
converters={'timestamp':convdate, 'step':convstep, 'dir': convdir})
if since:
dat = dat[dat['timestamp'] > since ]
dat2 = get_average_times(dat, 1, 2, data_column).transpose()
len_dat = 1 if len(dat.shape) == 0 else len(dat)
if dat2.size == 0:
print 'Did not load any record for {}'.format(name)
else:
print 'Loaded {0} records for {1} ({2[0]}x{2[1]} unique) ~= {3} run(s)'.format(
len(dat), name, dat2.shape, safe_div(len(dat),dat2.size))
len_dat, name, dat2.shape, safe_div(len_dat,dat2.size))
dat3 = dat2
return dat2
def read_results(prefix, name, dtype, data_column, since, splitted = no_split):
if splitted in (split_change_only, split_both):
result = {}
for change in kinds['changes']:
result.setdefault(change, {})
if splitted == split_both:
for strategy in kinds['strategies']:
result[change].setdefault(strategy, {})
new_name = '{0}_{1}_{2}'.format(change, strategy, name)
f = 'profiling/splitted/{0}_{1}.csv'.format(prefix, new_name)
result[change][strategy] = read_single_result(f, new_name, dtype, data_column, since)
else: # splitted == split_change_only
new_name = '{0}_{1}'.format(change, name)
f = 'profiling/splitted/{0}_{1}.csv'.format(prefix, new_name)
result[change] = read_single_result(f, new_name, dtype, data_column, since)
return result
else: # splitted = no_split
f = 'profiling/{0}-{1}-results.csv'.format(prefix, name)
return read_single_result(f, name, dtype, data_column, since)
```
%% Cell type:markdown id: tags:
# Need testing, if still working with change kinds
%% Cell type:code id: tags:
``` python
def read_gen_results(impl, since = None):
return read_results('gen', impl, ('datetime64[us]', int, int, float), 3, since, splitted = split_both)
def read_sol_results(solver, since = None):
return read_results('sol', solver, ('datetime64[us]', int, int, int, int, int, float, float), 7, since,
splitted = split_change_only)
def read_att_result(name = 'profiling/att-percentages.csv'):
def cdir(text):
prefix = text[:-4]
number = int(text[-3:])
if prefix == 'update':
return number
elif prefix == 'sw':
return 31+number
elif prefix == 'res':
return 35+number
elif prefix == '':
pass
else:
print 'Unknown prefix "{0}"'.format(prefix)
dat = np.genfromtxt(name, delimiter=',', names=True,
#dtype=(int, float, float, float, float, float, float, float),
converters={'dir': cdir})
dat.sort(axis=0)
print 'Loaded {0} attribute metrics'.format(len(dat))
return dat
```
%% Cell type:code id: tags:
``` python
def safe(a, i, start = 0):
try:
return a[start:,i]
except IndexError:
return np.zeros(a[:,0].size)
```
%% Cell type:markdown id: tags:
## ILP-Re-Generation and Solving Time
- ILP-Re-Generation describes the time it takes for the Scheme implementations to evaluate the attribute computing the ILP
- it does not include the time to read from and write to disk
- "Java" denotes the Java- and EMF-based generation
- ILP Solving describes the time it takes the solvers (GLPK and Gurobi in this case) to solve the generated ILP
- "GLPK (Java)" denotes the time to solve the ILP generated by the Java-based generation with GLPK. Its format was modifiert slightly to be accepted by GLPK, as it was originally generated for lp_solve
- "GLPK (Scheme)" denotes the time to solve the ILP generated by any scheme implementation with GLPK
- the plots show different system configurations
- a system configuration is given by "r x ( c \* i \* m )", which describes a system with *r* resources and *c* software components with *i* implementations having *m* modes each.
- for one such configuration the same experiment is run, i.e. the system is modified 7 times leading to the 7 steps, whereas only changes on hardware resources are made
- for the cases involving Java (both, generation and solving), the same time is used for every step, as the generation always starts from skretch. Further, the changes (e.g. change the value of a single hardware resource while generation) can not be reflected by the Java System Generator
- on the x-axis, the steps of this manipulation are shown
- the initial generation of the ILP (step zero) is only shown below in numbers, as it would skew the diagrams
%% Cell type:markdown id: tags:
Header = ['timestamp', 'dir', 'step', 'rows', 'cols', 'non-zero', 'ilp-sol', 'ti-ilp-sol']
%% Cell type:code id: tags:
``` python
racket_dats = read_gen_results('plt-r6rs', since = datetime(2015,8,17,0,0,0))
larceny_dats = read_gen_results('larceny', since = datetime(2015,6,12,0,0,0))
java_dats = read_gen_results('java')
```
%% Output
Loaded 328 records for update_normal_plt-r6rs (8x31 unique) ~= 1 run(s)
Loaded 256 records for update_flush_plt-r6rs (8x31 unique) ~= 1 run(s)
Loaded 496 records for update_noncached_plt-r6rs (8x31 unique) ~= 2 run(s)
Loaded 63 records for sw_normal_plt-r6rs (7x4 unique) ~= 2 run(s)
Loaded 28 records for sw_flush_plt-r6rs (7x4 unique) ~= 1 run(s)
Loaded 140 records for sw_noncached_plt-r6rs (7x13 unique) ~= 1 run(s)
Loaded 63 records for res_normal_plt-r6rs (7x4 unique) ~= 2 run(s)
Loaded 28 records for res_flush_plt-r6rs (7x4 unique) ~= 1 run(s)
Loaded 147 records for res_noncached_plt-r6rs (7x14 unique) ~= 1 run(s)
Did not load any record for update_normal_larceny
Loaded 264 records for update_normal_larceny (8x31 unique) ~= 1 run(s)
Did not load any record for update_flush_larceny
Did not load any record for update_noncached_larceny
Did not load any record for sw_normal_larceny
Did not load any record for sw_flush_larceny
Did not load any record for sw_noncached_larceny
Did not load any record for res_normal_larceny
Did not load any record for res_flush_larceny
Did not load any record for res_noncached_larceny
Did not load any record for update_normal_java
Did not load any record for update_flush_java
Did not load any record for update_noncached_java
Did not load any record for sw_normal_java
Did not load any record for sw_flush_java
Did not load any record for sw_noncached_java
Did not load any record for res_normal_java
Did not load any record for res_flush_java
Did not load any record for res_noncached_java
Loaded 88 records for update_normal_java (1x31 unique) ~= 2 run(s)
Loaded 1 records for update_flush_java (1x1 unique) ~= 1 run(s)
Loaded 1 records for update_noncached_java (1x1 unique) ~= 1 run(s)
Loaded 1 records for sw_normal_java (1x1 unique) ~= 1 run(s)
Loaded 1 records for sw_flush_java (1x1 unique) ~= 1 run(s)
Loaded 1 records for sw_noncached_java (1x1 unique) ~= 1 run(s)
Loaded 1 records for res_normal_java (1x1 unique) ~= 1 run(s)
Loaded 1 records for res_flush_java (1x1 unique) ~= 1 run(s)
Loaded 1 records for res_noncached_java (1x1 unique) ~= 1 run(s)
%% Cell type:code id: tags:
``` python
java_glpk_dats = read_sol_results('java', since = datetime(2015,6,22,0,0,0))
glpk_dats = read_sol_results('glpk')
gurobi_dats = read_sol_results('gurobi')
```
%% Output
Loaded 23 records for update_java (1x23 unique) ~= 1 run(s)
Did not load any record for sw_java
Did not load any record for res_java
Loaded 1656 records for update_glpk (8x23 unique) ~= 9 run(s)
Did not load any record for sw_glpk
Did not load any record for res_glpk
Did not load any record for update_gurobi
Did not load any record for sw_gurobi
Did not load any record for res_gurobi
Loaded 1 records for sw_glpk (1x1 unique) ~= 1 run(s)
Loaded 1 records for res_glpk (1x1 unique) ~= 1 run(s)
Loaded 216 records for update_gurobi (8x27 unique) ~= 1 run(s)
Loaded 1 records for sw_gurobi (1x1 unique) ~= 1 run(s)
Loaded 1 records for res_gurobi (1x1 unique) ~= 1 run(s)
%% Cell type:code id: tags:
``` python
p_ax_nr, p_line_def, p_col_def, p_label = 0, 1, 2, 3
p_gen_racket, p_gen_larceny, p_gen_java = 4, 5, 6
p_sol_glpk, p_sol_gurobi, p_sol_java_glpk = 4, 5, 6
```
%% Cell type:code id: tags:
``` python
def draw_gen(params):
def draw_gen(changeName, strategy, params):
# needed number of axes equals ax_nr+1 now
name = 'gen_{}'.format(changeName)
name = 'gen_{0}_{1}'.format(changeName, strategy)
f, ax_arr = plt.subplots(nrows = ax_nr+1, ncols = 3, sharex=True, sharey=True)
f.set_size_inches(25.5,3.5*(ax_nr+1))
one_plot = ax_arr.shape[1] == 1
f.patch.set_facecolor('none')
f.patch.set_alpha(0.0)
lines, labels = [], []
for p in params:
ax_tup = ax_arr if one_plot else ax_arr[p[p_ax_nr]]
ax_j = ax_tup[0]
ax_r = ax_tup[1]
ax_l = ax_tup[2]
# ax_j.set_ylim([0,50])
# ax_r.set_ylim([0,10])
# ax_l.set_ylim([0,10])
# x_g = np.array(xrange(1,len(p[1])+1)) # start at one, since first gen-time is cut
x_g = np.array(xrange(START_STEP,len(p[p_gen_racket])+START_STEP)) # start at zero
line_java = ax_j.plot(x_g, p[p_gen_java][0]*np.ones(len(p[p_gen_racket])), ls = p[p_line_def],
c = p[p_col_def], label = p[p_label])
line_racket = ax_r.plot(x_g, p[p_gen_racket], ls = p[p_line_def], c = p[p_col_def], label = p[p_label])
line_larceny = ax_l.plot(x_g, p[p_gen_larceny], ls = p[p_line_def], c = p[p_col_def], label = p[p_label])
ax_l.legend(loc='upper left', bbox_to_anchor=(1, 1.02))
lines.append(line_racket[0])
labels.append(p[p_label])
for ax in ax_arr if one_plot else ax_arr.flatten():
ax.set_ylabel('seconds')
ax.patch.set_alpha(1)
ax.patch.set_facecolor('white')
first_ax = ax_arr if one_plot else ax_arr[0]
plt.suptitle('ILP Generation Time', fontsize = 16)
first_ax[0].set_title('Java')
first_ax[1].set_title('Racket')
first_ax[2].set_title('Larceny')
# Fine-tune figure; make subplots close to each other and hide x ticks for
# all but bottom plots.
f.subplots_adjust(hspace=0.2)
plt.setp([a.get_xticklabels() for a in f.axes[:-3]], visible=False)
plt.setp([a.get_yticklabels() for a in f.axes], visible=True)
plt.savefig('doc/{}.pdf'.format(name), facecolor=f.get_facecolor(), edgecolor='none')
plt.savefig('{}.png'.format(name), facecolor=f.get_facecolor(), edgecolor='none')
plt.close()
```
%% Cell type:code id: tags:
``` python
def draw_sol(changeName, params):
name = 'sol_{}'.format(changeName)
f, ax_arr = plt.subplots(nrows = ax_nr+1, ncols = 3, sharex=True, sharey=True)
f.set_size_inches(25.5,3.5*(ax_nr+1))
one_plot = ax_arr.shape[1] == 1
f.patch.set_facecolor('none')
f.patch.set_alpha(0.0)
lines, labels = [], []
for p in params:
ax_tup = ax_arr if one_plot else ax_arr[p[p_ax_nr]]
ax_javaglpk = ax_tup[0]
ax_glpk = ax_tup[1]
ax_gurobi = ax_tup[2]
x = np.array(xrange(0,len(p[p_sol_glpk]))) # start at zero
line_javaglpk = ax_javaglpk.plot(x, p[p_sol_java_glpk][0]*np.ones(len(p[p_sol_glpk])), ls = p[p_line_def],
c = p[p_col_def], label = p[p_label])
line_glpk = ax_glpk.plot(x, p[p_sol_glpk], ls = p[p_line_def], c = p[p_col_def], label = p[p_label])
line_gurobi = ax_gurobi.plot(x, p[p_sol_gurobi], ls = p[p_line_def], c = p[p_col_def], label = p[p_label])
ax_gurobi.legend(loc='upper left', bbox_to_anchor=(1, 1.02))
lines.append(line_glpk[0])
labels.append(p[p_label])
for ax in ax_arr if one_plot else ax_arr.flatten():
ax.set_ylabel('seconds')
ax.patch.set_alpha(1)
ax.patch.set_facecolor('white')
first_ax = ax_arr if one_plot else ax_arr[0]
plt.suptitle('ILP Solving Time', fontsize = 16)
first_ax[0].set_title('GLPK (Java)')
first_ax[1].set_title('GLPK (Scheme)')
first_ax[2].set_title('Gurobi (Scheme)')
# Fine-tune figure; make subplots close to each other and hide x ticks for
# all but bottom plots.
f.subplots_adjust(hspace=0.2)
plt.setp([a.get_xticklabels() for a in f.axes[:-3]], visible=False)
plt.setp([a.get_yticklabels() for a in f.axes], visible=True)
plt.savefig('doc/{}.pdf'.format(name))
plt.savefig('{}.png'.format(name))
plt.close()
```
%% Cell type:code id: tags:
``` python
def draw_comp_sol(changeName, params):
name = 'comp_sol_{}'.format(changeName)
indices = [3, 12, 14, 19]
for i in indices:
p = params[i]
line_javaglpk = plt.plot(x, p[p_sol_java_glpk][0]*np.ones(len(p[p_sol_glpk])), ls = linestyles[0],
c = p[p_col_def], label = p[p_label])
line_glpk = plt.plot(x, p[p_sol_glpk], ls = p[p_line_def], c = p[p_col_def])
line_gurobi = plt.plot(x, p[p_sol_gurobi], ls = p[p_line_def], c = p[p_col_def])
plt.legend(loc = 'right')
plt.ylabel('seconds')
plt.suptitle('ILP Solving Time - Comparison', fontsize = 16)
plt.savefig('doc/{}.pdf'.format(name))
plt.savefig('{}.png'.format(name))
plt.close()
```
%% Cell type:code id: tags:
``` python
START_STEP, MAX_PLOTS_IN_ONE = 0, 7
for change in kinds['changes']:
print 'Change = {}'.format(change)
for strategy in kinds['strategies']:
print 'Stategy = {}'.format(strategy)
current_plot, ax_nr, last_res = 0, 0, -1
gen_params = []
for i in xrange(len(specs)):
current_res = specs[i][2]
current_plot += 1
if current_plot > MAX_PLOTS_IN_ONE and last_res != current_res:
ax_nr += 1
current_plot = 0
# params.append([ax_nr, safe(racket_dats[change][strategy],i,START_STEP), safe(larceny_dats[change][strategy],i,START_STEP),
# safe(glpk_dat[change][strategy],i), safe(gurobi_dat[change][strategy],i),
# safe(glpk_dats[change][strategy],i), safe(gurobi_dats[change][strategy],i),
# line_def[i], color_def[i], '{2:d} x ({4}*{5}*{6})'.format(*specs[i]),
# safe(java_dat[change][strategy],i), safe(java_glpk_dat[change][strategy],i)])
# safe(java_dats[change][strategy],i), safe(java_glpk_dats[change][strategy],i)])
gen_params.append([ax_nr, line_def[i], color_def[i], '{2:d} x ({4}*{5}*{6})'.format(*specs[i]),
safe(racket_dats[change][strategy],i,START_STEP),
safe(larceny_dats[change][strategy],i,START_STEP),
safe(java_dat[change][strategy],i)])
safe(java_dats[change][strategy],i)])
last_res = current_res
try:
draw_gen(gen_params)
draw_gen(change, strategy, gen_params)
except:
print 'Error while drawing gen in {0}-{1}'.format(change, strategy)
sol_params = []
for i in xrange(len(specs)):
current_res = specs[i][2]
current_plot += 1
if current_plot > MAX_PLOTS_IN_ONE and last_res != current_res:
ax_nr += 1
current_plot = 0
sol_params.append([ax_nr, line_def[i], color_def[i], '{2:d} x ({4}*{5}*{6})'.format(*specs[i]),
safe(glpk_dat[change][strategy],i),
safe(gurobi_dat[change][strategy],i),
safe(java_glpk_dat[change][strategy],i)])
safe(glpk_dats[change][strategy],i),
safe(gurobi_dats[change][strategy],i),
safe(java_glpk_dats[change][strategy],i)])
try:
draw_sol(params)
draw_sol(change, params)
except:
print 'Error while drawing sol in {0}-{1}'.format(change, strategy)
try:
draw_comp_sol(params)
draw_comp_sol(change, params)
except:
print 'Error while drawing comp-sol in {0}-{1}'.format(change, strategy)
```
%% Output
Change = update
Stategy = normal
Stategy = flush
---------------------------------------------------------------------------
IndexError Traceback (most recent call last)
<ipython-input-68-e8120160290e> in <module>()
16 gen_params.append([ax_nr, line_def[i], color_def[i], '{2:d} x ({4}*{5}*{6})'.format(*specs[i]),
17 safe(racket_dats[change][strategy],i,START_STEP),
---> 18 safe(larceny_dats[change][strategy],i,START_STEP),
19 safe(java_dat[change][strategy],i)])
20 last_res = current_res
<ipython-input-168-4ac34798cab4> in <module>()
18 gen_params.append([ax_nr, line_def[i], color_def[i], '{2:d} x ({4}*{5}*{6})'.format(*specs[i]),
19 safe(racket_dats[change][strategy],i,START_STEP),
---> 20 safe(larceny_dats[change][strategy],i,START_STEP),
21 safe(java_dats[change][strategy],i)])
22 last_res = current_res
<ipython-input-10-27c199385393> in safe(a, i, start)
3 return a[start:,i]
4 except IndexError:
----> 5 return np.zeros(a[:,0].size)
IndexError: too many indices for array
%% Cell type:code id: tags:
``` python
larceny_dats
```
%% Output
{u'res': {u'flush': array([], dtype=float64),
u'noncached': array([], dtype=float64),
u'normal': array([], dtype=float64)},
u'sw': {u'flush': array([], dtype=float64),
u'noncached': array([], dtype=float64),
u'normal': array([], dtype=float64)},
u'update': {u'flush': array([], dtype=float64),
u'noncached': array([], dtype=float64),
u'normal': array([[ 0.17 , 0.187 , 1.766 , 11.3 , 0.3 , 0.403 ,
3.304 , 22.73 , 1.439 , 2.375 , 3.521 , 6.308 ,
10.359 , 12.319 , 15.182 , 18.13 , 22.5205, 27.992 ,
32.143 , 25.217 , 30.606 , 38.144 , 45.88 , 4.294 ,
9.545 , 16.402 , 24.272 , 0.362 , 0.711 , 1.518 , 3.948 ],
[ 0.5 , 0.46 , 0.223 , 0.916 , 0.9 , 0.123 ,
0.446 , 2.267 , 0.257 , 0.488 , 0.852 , 1.224 ,
1.964 , 2.78 , 2.6 , 3.274 , 4.2555, 5.503 ,
4.899 , 4.107 , 4.98 , 6.655 , 7.133 , 0.958 ,
1.695 , 2.673 , 3.984 , 0.68 , 0.142 , 0.239 , 0.649 ],
[ 0.5 , 0.45 , 0.228 , 0.915 , 0.1 , 0.117 ,
0.441 , 2.266 , 0.261 , 0.489 , 0.879 , 1.255 ,
1.953 , 2.79 , 2.602 , 3.252 , 3.577 , 5.444 ,
4.878 , 4.54 , 4.959 , 6.679 , 7.165 , 0.949 ,
1.507 , 2.679 , 3.977 , 0.73 , 0.141 , 0.241 , 0.65 ],
[ 0.5 , 0.43 , 0.225 , 0.902 , 0.11 , 0.115 ,
0.443 , 2.256 , 0.25 , 0.488 , 0.858 , 1.209 ,
1.958 , 2.93 , 2.636 , 3.272 , 3.5695, 5.444 ,
4.8705, 4.77 , 4.979 , 6.732 , 7.151 , 0.937 ,
1.52 , 2.692 , 3.995 , 0.68 , 0.147 , 0.231 , 0.675 ],
[ 0.4 , 0.28 , 0.35 , 0.83 , 0.6 , 0.64 ,
0.8 , 0.225 , 0.129 , 0.222 , 0.263 , 0.525 ,
0.714 , 0.919 , 1.202 , 1.385 , 1.607 , 1.866 ,
1.475 , 1.88 , 2.163 , 2.395 , 2.83 , 0.326 ,
0.862 , 1.473 , 2.274 , 0.26 , 0.62 , 0.178 , 0.253 ],
[ 0.5 , 0.39 , 0.45 , 0.102 , 0.1 , 0.87 ,
0.171 , 0.27 , 0.162 , 0.272 , 0.359 , 0.636 ,
0.952 , 1.311 , 1.965 , 2.199 , 2.3975, 3.42 ,
2.397 , 2.855 , 2.935 , 3.413 , 4.4 , 0.534 ,
1.105 , 2.147 , 2.773 , 0.44 , 0.8 , 0.145 , 0.381 ],
[ 0.5 , 0.39 , 0.14 , 0.153 , 0.1 , 0.12 ,
0.103 , 0.257 , 0.186 , 0.253 , 0.42 , 0.852 ,
0.945 , 1.319 , 1.926 , 1.779 , 2.23 , 2.445 ,
2.262 , 2.565 , 2.833 , 3.318 , 3.743 , 0.456 ,
1.298 , 1.967 , 2.767 , 0.46 , 0.77 , 0.162 , 0.334 ],
[ 0.4 , 0.4 , 0.51 , 0.144 , 0.9 , 0.8 ,
0.135 , 0.36 , 0.162 , 0.443 , 0.343 , 0.681 ,
0.933 , 1.357 , 1.623 , 1.834 , 2.4315, 2.626 ,
2.098 , 2.467 , 2.755 , 4.685 , 3.96 , 0.542 ,
1.199 , 2.159 , 2.642 , 0.38 , 0.93 , 0.151 , 0.331 ]])}}
%% Cell type:markdown id: tags:
## Times for initial generation of the ILP
%% Cell type:code id: tags:
``` python
(racket_dat[0], larceny_dat[0])
```
%% Output
(array([ 0.82 , 0.239, 2.284, 19.2 , 0.21 , 0.319, 3.414,
27.39 , 1.433, 2.482, 3.342, 6.396, 10.884, 12.314,
14.289, 15.988, 21.376, 26.704, 32.28 , 24.226, 29.95 ,
36.352, 43.842, 3.219, 7.972, 12.263, 17.145, 0.352,
0.763, 2.226, 9.606]),
array([ 0.17 , 0.187 , 1.766 , 11.3 , 0.3 , 0.403 ,
3.304 , 22.73 , 1.439 , 2.375 , 3.521 , 6.308 ,
10.359 , 12.319 , 15.182 , 18.13 , 22.5205, 27.992 ,
32.143 , 25.217 , 30.606 , 38.144 , 45.88 , 4.294 ,
9.545 , 16.402 , 24.272 , 0.362 , 0.711 , 1.518 , 3.948 ]))
%% Cell type:markdown id: tags:
## Attribute metrics
- X = {normal, flushed, noncached}
- (X1,X2) = {(normal, flushed), (normal, noncached), (flushed, noncached)}
%% Cell type:code id: tags:
``` python
att_dat = read_att_result()
att_dat.dtype
```
%% Output
Loaded 39 attribute metrics
dtype([('dir', '<f8'), ('normalBaseline', '<f8'), ('flushedBaseline', '<f8'), ('noncachedBaseline', '<f8'), ('ratioNormalToFlushed', '<f8'), ('ratioNormalToNoncached', '<f8'), ('ratioFlushedToNoncached', '<f8'), ('speedupNormalToFlushed', '<f8'), ('speedupNormalToNoncached', '<f8'), ('speedupFlushedToNoncached', '<f8')])
%% Cell type:markdown id: tags:
1) {X}basline: `total.X.computed / total.X.called`
- the baseline of the method X
%% Cell type:code id: tags:
``` python
width = 0.5
x = att_dat['dir']
for n, (l, c) in enumerate((('noncached', 'grey'), ('flushed', 'g'), ('normal', 'b'))):
y = att_dat[l + 'Baseline']
#plt.plot(x, y, label = l, color = c)
plt.bar(x*4*width + n*width, y, width, label = l, color = c)
plt.legend(loc = 'best')
plt.ylabel('% computed')
plt.suptitle('Attribute Baselines', fontsize = 16)
plt.savefig('doc/att_bl.pdf')
!pdfcrop doc/att_bl.pdf doc/att_bl_cropped.pdf > /dev/null
plt.savefig('att_bl.png')
plt.close()
```
%% Cell type:markdown id: tags:
2) ratio{X1}To{X2}: `total.{X1}.computed / total.{X2}.called`
- the efficiency of the incremental approach in comparison to the method X2,
i.e. the ratio between actual work done in X1 compared to possible work done with method X2
%% Cell type:code id: tags:
``` python
x = att_dat['dir']
for n, (name, c, ls) in enumerate((('FlushedToNoncached', 'orange', '--'),
('NormalToFlushed', 'purple', '-'),
('NormalToNoncached', 'r', '-.'))):
y = att_dat['ratio' + name]
plt.plot(x, y, label = name.replace('To', r'$\rightarrow$'), c = c, ls = ls)
#plt.bar(x*3*width + n*width, y, width, label = l, color = c)
plt.legend(loc = 'best')
plt.ylabel('%')
plt.suptitle('Attribute Ratios', fontsize = 16)
plt.savefig('doc/att_r.pdf')
!pdfcrop doc/att_r.pdf doc/att_r_cropped.pdf > /dev/null
plt.savefig('att_r.png')
plt.close()
```
%% Cell type:markdown id: tags:
3) speedup{X1}To{X2}: `(total.{X2}.computed / total.{X2}.called) - (total.{X1}.computed / total.{X2}.called)`
- = `baseline({X2}) - ratio({X1}, {X2})`
- the "speed-up" of the incremental approach (normal or flushed) in comparison to the method X2
%% Cell type:code id: tags:
``` python
x = att_dat['dir']
for n, (l, c) in enumerate((('NormalToNoncached', 'r'),
('FlushedToNoncached', 'orange'),
('NormalToFlushed', 'yellow'))):
y = att_dat['speedup' + l]
plt.plot(x, y, label = l.replace('To', r'$\rightarrow$'), color = c)
#plt.bar(x*3*width + n*width, y, width, label = l, color = c)
plt.legend(loc = 'best')
plt.ylabel('%')
plt.suptitle('Attribute Speed-Ups', fontsize = 16)
plt.savefig('doc/att_sp.pdf')
!pdfcrop doc/att_sp.pdf doc/att_sp_cropped.pdf > /dev/null
plt.savefig('att_sp.png')
plt.close()
```
%% Cell type:markdown id: tags:
## Boxplots for attribute measures
%% Cell type:code id: tags:
``` python
att_totals = {}
for change in d['changes']:
att_totals[change] = read_att_result(name = 'profiling/splitted/att-percentages_{}.csv'.format(change))
```
%% Output
Loaded 31 attribute metrics
Loaded 4 attribute metrics
Loaded 4 attribute metrics
%% Cell type:code id: tags:
``` python
PRINT_NONCACHED = True
fig, axes = plt.subplots(ncols=len(d['changes']))
l = ['normal', 'flushed']
if PRINT_NONCACHED:
l.append('noncached')
for i, change in enumerate(d['changes']):
att_dat = att_totals[change]
axes[i].boxplot([att_dat[name + 'Baseline'] for name in l], labels = [name[0]+name[3] for name in l])
axes[i].set_title(change, fontsize=10)
if i > 0:
axes[i].set_yticklabels([])
if PRINT_NONCACHED:
for ax in axes.flatten():
ax.set_ylim([0,1.05])
plt.suptitle('Attribute Baselines', fontsize = 16)
plt.savefig('doc/att_box_bl.pdf')
!pdfcrop doc/att_box_bl.pdf doc/att_box_bl_cropped.pdf > /dev/null
plt.savefig('att_box_bl.png')
plt.close()
```
......
ilp_measurement.py
+ 14
− 1
View file @ 4be698eb
......@@ -297,8 +297,18 @@ def clean(dryrun = False):
total += secure_remove({'profiling/splitted': ['*.csv']}, globbing = True, dryrun = dryrun)
print 'Removed {} files.'.format(total)
dummy_values = {'timestamp': '1970-01-01T00:00:00.00', 'dir': 'dummy-001', 'step': '01-dummy', 'attname': 'dummyatt'}
@task(name = 'distinction-of-changes')
def change_distinction():
def maybe_insert_dummy(f):
with open(f, 'a+') as fd:
header = next(fd)
if not next(fd, False):
# insert dummy data
keys = (key.strip() for key in header.split(','))
values = [dummy_values.get(key, '0') for key in keys]
fd.write(','.join(values))
with open(change_kinds) as fd:
d = json.load(fd)
unnormal = '-v -e ' + ' -e '.join((c for c in d['strategies'] if c != 'normal'))
......@@ -313,6 +323,7 @@ def change_distinction():
sol_target = 'profiling/splitted/sol_{0}_{1}.csv'.format(change, sol_name)
local_quiet('tail -n +2 profiling/sol-header > {0}'.format(sol_target))
local_quiet('tail -n +2 {0} | grep -e {1} | cat >> {2}'.format(f, change, sol_target))
maybe_insert_dummy(sol_target)
# att percentages (only per change kinds)
f = 'profiling/att-percentages.csv'
......@@ -320,6 +331,7 @@ def change_distinction():
local_quiet('head -n 1 profiling/att-percentages.csv > {}'.format(target))
local_quiet('tail -n +2 {0} | grep -e {1} | cat >> {2}'.format(
f, change, target))
maybe_insert_dummy(target)
for strategy in d['strategies']:
sys.stdout.write('.')
......@@ -330,13 +342,14 @@ def change_distinction():
shutil.copy('profiling/gen-header', gen_target)
local_quiet('tail -n +2 {0} | grep -e {1} | grep {2} | cat >> {3}'.format(
f, change, get_strategy_pattern(strategy), gen_target))
maybe_insert_dummy(gen_target)
# att totals
f = 'profiling/all-att-results.csv'
target = 'profiling/splitted/att_{0}_{1}.csv'.format(change, strategy)
local_quiet('head -n 1 profiling/att-totals.csv > {}'.format(target))
local_quiet('tail -n +2 {0} | grep -e {1} | grep {2} | cat >> {3}'.format(
f, change, get_strategy_pattern(strategy), target))
maybe_insert_dummy(target)
@task(name = 'prepare-noncached')
def prepare_noncached():
......
scheme.properties
+ 2
− 2
View file @ 4be698eb
......@@ -5,6 +5,6 @@ timing = 0
log.info = 1
log.debug = 0
measure.lp.write = 0
measure.profiling = 0
measure.flush = 1
measure.profiling = 1
measure.flush = 0
measure.non-chached = 0
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment