Skip to content
Snippets Groups Projects
Commit ae4f7bf4 authored by René Schöne's avatar René Schöne
Browse files

Increase font size in diagrams.

parent 38c7f3f4
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
Showing
with 16 additions and 9 deletions
ilp-measurement.ipynb
+ 16
− 9
View file @ ae4f7bf4
%% Cell type:code id: tags:
``` python
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.axes, matplotlib.ticker
import matplotlib.patches as mpatches
%matplotlib inline
plt.ioff()
```
%% Cell type:code id: tags:
``` python
from glob import glob
from datetime import datetime
from itertools import *
from operator import itemgetter
import json, sys, traceback, os, functools, traceback
```
%% Cell type:code id: tags:
``` python
_pngpath, _pdfpath = 'pngs/', 'pdfs/'
if not os.path.exists(_pngpath):
os.mkdir(_pngpath)
if not os.path.exists(_pdfpath):
os.mkdir(_pdfpath)
def pngpath(filename):
return _pngpath + filename + '.png'
def pdfpath(filename):
return _pdfpath + filename + '.pdf'
```
%% Cell type:code id: tags:
``` python
spec_files = glob('profiling/*/specs')
spec_files.sort()
specs = {os.path.basename(os.path.dirname(f)):np.genfromtxt(f, delimiter=' ', dtype=int)
for f in spec_files}
```
%% Cell type:code id: tags:
``` python
linestyles = ["-","--","-.",":"]
colors = ('blue','black','red','orange','green','magenta','yellow', 'cyan', 'purple', 'firebrick')
colorcycle = cycle(colors)
lastpe = False
line_def = []
color_def = []
## Currently disabled due to fragile and unused output
if False:
for spec in specs:
pe, comp, impl, mode = (spec[2], spec[4], spec[5], spec[6])
if not lastpe or lastpe != pe:
color = next(colorcycle)
linecycle = cycle(linestyles)
line = next(linecycle)
lastpe = pe
line_def.append(line)
color_def.append(color)
```
%% Cell type:code id: tags:
``` python
def set_keys(*indices):
"""Returns a function that returns a tuple of key values"""
def get_keys(seq, indices=indices):
keys = []
for i in indices:
keys.append(seq[i])
return tuple(keys)
return get_keys
```
%% Cell type:code id: tags:
``` python
def get_average_times(dat, dirCol, stepCol, timeCol):
if dat.size == 1:
return np.array([np.array([dat.item()[timeCol]])])
dat.sort(order=['dir', 'step'])
result = {}
for (c_dir, c_step), rows in groupby(dat, key=set_keys('dir','step')):
#print c_dir, c_step, rows
c_dir, c_step, total_time, counter = None, None, 0, 0
for row in rows:
if not c_dir:
c_dir = row[dirCol]
if not c_step:
c_step = row[stepCol]
total_time+=row[timeCol]
counter += 1
result.setdefault(c_dir, []).append([c_step, total_time*1.0/counter])
result2 = []
for c_dir, rows in result.iteritems():
inner = []
result2.append([row[1] for row in rows])
# for row in rows:
# inner.append(row[1])
# result2.append(inner)
return np.array([np.array(rows) for rows in result2])
```
%% Cell type:code id: tags:
``` python
def is_axis(a):
return not type(a) is np.ndarray
```
%% Cell type:code id: tags:
``` python
with open('profiling/kinds.json') as fd:
kinds = json.load(fd)
print 'Kinds: {}'.format(kinds)
```
%% Output
Kinds: {u'strategies': [u'normal', u'flush', u'noncached'], u'changes': [u'update', u'sw', u'res', u'complex', u'mixed']}
%% Cell type:code id: tags:
``` python
no_split, split_change_only, split_both = 0, 1, 2
dat, dat2, dat3 = None, None, None
def read_single_result(f, name, dtype, data_column, since):
global dat, dat2, dat3
def convdate(text):
return datetime.strptime(text, '%Y-%m-%dT%H:%M:%S.%f')
def convdir(text):
return int(text[-3:])
def convstep(text):
return int(text[0:2])
def safe_div(a, b):
return a/b if b>0 else "{}!".format(a)
dat = np.genfromtxt(f, delimiter=',', names=True, dtype=dtype,
converters={'timestamp':convdate, 'step':convstep, 'dir': convdir})
if since:
dat = dat[dat['timestamp'] > since ]
dat2 = get_average_times(dat, 1, 2, data_column).transpose()
len_dat = 1 if len(dat.shape) == 0 else len(dat)
if dat2.size == 0:
print 'Did not load any record for {}'.format(name)
else:
if len_dat > 1:
print 'Loaded {0} record(s) for {1} ({2[0]}x{2[1]} unique) ~= {3} run(s)'.format(
len_dat, name, dat2.shape, safe_div(len_dat,dat2.size))
else:
print 'Loaded no data for {}'.format(name)
dat3 = dat2
return dat2
def read_results(prefix, name, dtype, data_column, since, splitted = no_split):
if splitted in (split_change_only, split_both):
result = {}
for change in kinds['changes']:
result.setdefault(change, {})
if splitted == split_both:
for strategy in kinds['strategies']:
result[change].setdefault(strategy, {})
new_name = '{0}_{1}_{2}'.format(change, strategy, name)
f = 'profiling/splitted/{0}_{1}.csv'.format(prefix, new_name)
result[change][strategy] = read_single_result(f, new_name, dtype, data_column, since)
else: # splitted == split_change_only
new_name = '{0}_{1}'.format(change, name)
f = 'profiling/splitted/{0}_{1}.csv'.format(prefix, new_name)
result[change] = read_single_result(f, new_name, dtype, data_column, since)
return result
else: # splitted = no_split
f = 'profiling/{0}-{1}-results.csv'.format(prefix, name)
return read_single_result(f, name, dtype, data_column, since)
```
%% Cell type:code id: tags:
``` python
def read_gen_results(impl, since = None):
return read_results('gen', impl, ('datetime64[us]', int, int, float), 3, since, splitted = split_both)
def read_sol_results(solver, since = None):
return read_results('sol', solver, ('datetime64[us]', int, int, int, int, int, float, float), 7, since,
splitted = split_change_only)
```
%% Cell type:code id: tags:
``` python
def safe(a, i, start = 0):
try:
return a[start:,i]
except IndexError:
return np.zeros(a[:,0].size)
def es(y, x):
""" Ensure size of y compared to x """
if y.shape[0] != x.shape[0]:
y = np.ones(x.shape[0]) * y[0]
return y
```
%% Cell type:markdown id: tags:
## ILP-Re-Generation and Solving Time
- ILP-Re-Generation describes the time it takes for the Scheme implementations to evaluate the attribute computing the ILP
- it does not include the time to read from and write to disk
- "Java" denotes the Java- and EMF-based generation
- ILP Solving describes the time it takes the solvers (GLPK and Gurobi in this case) to solve the generated ILP
- "GLPK (Java)" denotes the time to solve the ILP generated by the Java-based generation with GLPK. Its format was modifiert slightly to be accepted by GLPK, as it was originally generated for lp_solve
- "GLPK (Scheme)" denotes the time to solve the ILP generated by any scheme implementation with GLPK
- the plots show different system configurations
- a system configuration is given by "r x ( c \* i \* m )", which describes a system with *r* resources and *c* software components with *i* implementations having *m* modes each.
- for one such configuration the same experiment is run, i.e. the system is modified 7 times leading to the 7 steps, whereas only changes on hardware resources are made
- for the cases involving Java (both, generation and solving), the same time is used for every step, as the generation always starts from skretch. Further, the changes (e.g. change the value of a single hardware resource while generation) can not be reflected by the Java System Generator
- on the x-axis, the steps of this manipulation are shown
- the initial generation of the ILP (step zero) is only shown below in numbers, as it would skew the diagrams
%% Cell type:markdown id: tags:
Header = ['timestamp', 'dir', 'step', 'rows', 'cols', 'non-zero', 'ilp-sol', 'ti-ilp-sol']
%% Cell type:code id: tags:
``` python
racket_dats = read_gen_results('plt-r6rs', since = datetime(2015,8,17,0,0,0))
larceny_dats = read_gen_results('larceny', since = datetime(2015,6,12,0,0,0))
java_dats = read_gen_results('java')
```
%% Output
Loaded 1584 record(s) for update_normal_plt-r6rs (8x31 unique) ~= 6 run(s)
Loaded 504 record(s) for update_flush_plt-r6rs (8x31 unique) ~= 2 run(s)
Loaded 600 record(s) for update_noncached_plt-r6rs (8x31 unique) ~= 2 run(s)
Loaded 588 record(s) for sw_normal_plt-r6rs (7x31 unique) ~= 2 run(s)
Loaded 245 record(s) for sw_flush_plt-r6rs (7x31 unique) ~= 1 run(s)
Loaded 602 record(s) for sw_noncached_plt-r6rs (7x31 unique) ~= 2 run(s)
Loaded 605 record(s) for res_normal_plt-r6rs (7x31 unique) ~= 2 run(s)
Loaded 245 record(s) for res_flush_plt-r6rs (7x31 unique) ~= 1 run(s)
Loaded 609 record(s) for res_noncached_plt-r6rs (7x31 unique) ~= 2 run(s)
Loaded 242 record(s) for complex_normal_plt-r6rs (11x11 unique) ~= 2 run(s)
Loaded 121 record(s) for complex_flush_plt-r6rs (11x11 unique) ~= 1 run(s)
Loaded 121 record(s) for complex_noncached_plt-r6rs (11x11 unique) ~= 1 run(s)
Loaded 803 record(s) for mixed_normal_plt-r6rs (100x2 unique) ~= 4 run(s)
Loaded 359 record(s) for mixed_flush_plt-r6rs (100x2 unique) ~= 1 run(s)
Loaded 200 record(s) for mixed_noncached_plt-r6rs (100x2 unique) ~= 1 run(s)
Loaded 512 record(s) for update_normal_larceny (8x31 unique) ~= 2 run(s)
Loaded no data for update_flush_larceny
Loaded no data for update_noncached_larceny
Loaded 217 record(s) for sw_normal_larceny (7x31 unique) ~= 1 run(s)
Loaded no data for sw_flush_larceny
Loaded no data for sw_noncached_larceny
Loaded 217 record(s) for res_normal_larceny (7x31 unique) ~= 1 run(s)
Loaded no data for res_flush_larceny
Loaded no data for res_noncached_larceny
Loaded 121 record(s) for complex_normal_larceny (11x11 unique) ~= 1 run(s)
Loaded no data for complex_flush_larceny
Loaded no data for complex_noncached_larceny
Loaded no data for mixed_normal_larceny
Loaded no data for mixed_flush_larceny
Loaded no data for mixed_noncached_larceny
Loaded 88 record(s) for update_normal_java (1x31 unique) ~= 2 run(s)
Loaded no data for update_flush_java
Loaded no data for update_noncached_java
Loaded no data for sw_normal_java
Loaded no data for sw_flush_java
Loaded no data for sw_noncached_java
Loaded no data for res_normal_java
Loaded no data for res_flush_java
Loaded no data for res_noncached_java
Loaded no data for complex_normal_java
Loaded no data for complex_flush_java
Loaded no data for complex_noncached_java
Loaded no data for mixed_normal_java
Loaded no data for mixed_flush_java
Loaded no data for mixed_noncached_java
%% Cell type:code id: tags:
``` python
java_glpk_dats = read_sol_results('java', since = datetime(2015,6,22,0,0,0))
try:
glpk_dats = read_sol_results('glpk')
except:
print(traceback.format_exc())
gurobi_dats = read_sol_results('gurobi')
```
%% Output
Loaded 23 record(s) for update_java (1x23 unique) ~= 1 run(s)
Loaded no data for sw_java
Loaded no data for res_java
Loaded no data for complex_java
Loaded no data for mixed_java
Loaded 1928 record(s) for update_glpk (8x31 unique) ~= 7 run(s)
Loaded 241 record(s) for sw_glpk (7x31 unique) ~= 1 run(s)
Traceback (most recent call last):
File "<ipython-input-14-a9613a72d048>", line 3, in <module>
glpk_dats = read_sol_results('glpk')
File "<ipython-input-11-3738473b7378>", line 5, in read_sol_results
splitted = split_change_only)
File "<ipython-input-10-b4f10b934785>", line 44, in read_results
result[change] = read_single_result(f, new_name, dtype, data_column, since)
File "<ipython-input-10-b4f10b934785>", line 24, in read_single_result
len_dat, name, dat2.shape, safe_div(len_dat,dat2.size))
IndexError: tuple index out of range
Loaded 216 record(s) for update_gurobi (8x27 unique) ~= 1 run(s)
Loaded no data for sw_gurobi
Loaded no data for res_gurobi
Loaded no data for complex_gurobi
Loaded no data for mixed_gurobi
%% Cell type:code id: tags:
``` python
p_ax_nr, p_line_def, p_col_def, p_label = 0, 1, 2, 3
p_gen_racket, p_gen_larceny, p_gen_java = 4, 5, 6
p_sol_glpk, p_sol_gurobi, p_sol_java_glpk = 4, 5, 6
```
%% Cell type:code id: tags:
``` python
def draw_gen(changeName, strategy, params):
try:
name = 'gen_{0}_{1}'.format(changeName, strategy)
# needed number of axes equals ax_nr+1 now
f, ax_arr = plt.subplots(nrows = ax_nr+1, ncols = 3, sharex=True, sharey=True)
f.set_size_inches(25.5,3.5*(ax_nr+1))
one_plot = ax_arr.shape[1] == 1
f.patch.set_facecolor('none')
f.patch.set_alpha(0.0)
lines, labels = [], []
for p in params:
ax_tup = ax_arr if one_plot else ax_arr[p[p_ax_nr]]
ax_j = ax_tup[0]
ax_r = ax_tup[1]
ax_l = ax_tup[2]
# ax_j.set_ylim([0,50])
# ax_r.set_ylim([0,10])
# ax_l.set_ylim([0,10])
# x_g = np.array(xrange(1,len(p[1])+1)) # start at one, since first gen-time is cut
x_g = np.array(xrange(START_STEP,len(p[p_gen_racket])+START_STEP)) # start at zero
line_java = ax_j.plot(x_g, p[p_gen_java][0]*np.ones(len(p[p_gen_racket])), ls = p[p_line_def],
c = p[p_col_def], label = p[p_label])
line_racket = ax_r.plot(x_g, es(p[p_gen_racket], x_g), ls = p[p_line_def],
c = p[p_col_def], label = p[p_label])
line_larceny = ax_l.plot(x_g, es(p[p_gen_larceny], x_g), ls = p[p_line_def],
c = p[p_col_def], label = p[p_label])
ax_l.legend(loc='upper left', bbox_to_anchor=(1, 1.02))
lines.append(line_racket[0])
labels.append(p[p_label])
for ax in ax_arr if one_plot else ax_arr.flatten():
ax.set_ylabel('seconds')
ax.patch.set_alpha(1)
ax.patch.set_facecolor('white')
# first_ax = ax_arr if one_plot else ax_arr[0]
# plt.suptitle('ILP Generation Time', fontsize = 16)
# first_ax[0].set_title('Java')
# first_ax[1].set_title('Racket')
# first_ax[2].set_title('Larceny')
# Fine-tune figure; make subplots close to each other and hide x ticks for
# all but bottom plots.
f.subplots_adjust(hspace=0.2)
plt.setp([a.get_xticklabels() for a in f.axes[:-3]], visible=False)
plt.setp([a.get_yticklabels() for a in f.axes], visible=True)
plt.savefig(pdfpath(name), facecolor=f.get_facecolor(), edgecolor='none')
plt.savefig(pngpath(name), facecolor=f.get_facecolor(), edgecolor='none')
except Exception as e:
print 'Error while drawing gen in {0}-{1}: {2}'.format(change, strategy, e)
traceback.print_exc(file=sys.stdout)
finally:
plt.close()
```
%% Cell type:code id: tags:
``` python
def draw_sol(changeName, params):
try:
name = 'sol_{}'.format(changeName)
f, ax_arr = plt.subplots(nrows = ax_nr+1, ncols = 3, sharex=True, sharey=True)
f.set_size_inches(25.5,3.5*(ax_nr+1))
one_plot = ax_arr.shape[1] == 1
f.patch.set_facecolor('none')
f.patch.set_alpha(0.0)
lines, labels = [], []
for p in params:
ax_tup = ax_arr if one_plot else ax_arr[p[p_ax_nr]]
ax_javaglpk = ax_tup[0]
ax_glpk = ax_tup[1]
ax_gurobi = ax_tup[2]
x = np.array(xrange(0,len(p[p_sol_glpk]))) # start at zero
line_javaglpk = ax_javaglpk.plot(x, p[p_sol_java_glpk][0]*np.ones(len(p[p_sol_glpk])), ls = p[p_line_def],
c = p[p_col_def], label = p[p_label])
line_glpk = ax_glpk.plot(x, es(p[p_sol_glpk],x), ls = p[p_line_def],
c = p[p_col_def], label = p[p_label])
line_gurobi = ax_gurobi.plot(x, es(p[p_sol_gurobi],x), ls = p[p_line_def],
c = p[p_col_def], label = p[p_label])
ax_gurobi.legend(loc='upper left', bbox_to_anchor=(1, 1.02))
lines.append(line_glpk[0])
labels.append(p[p_label])
for ax in ax_arr if one_plot else ax_arr.flatten():
ax.set_ylabel('seconds')
ax.patch.set_alpha(1)
ax.patch.set_facecolor('white')
# first_ax = ax_arr if one_plot else ax_arr[0]
# plt.suptitle('ILP Solving Time', fontsize = 16)
# first_ax[0].set_title('GLPK (Java)')
# first_ax[1].set_title('GLPK (Scheme)')
# first_ax[2].set_title('Gurobi (Scheme)')
# Fine-tune figure; make subplots close to each other and hide x ticks for
# all but bottom plots.
f.subplots_adjust(hspace=0.2)
plt.setp([a.get_xticklabels() for a in f.axes[:-3]], visible=False)
plt.setp([a.get_yticklabels() for a in f.axes], visible=True)
plt.savefig(pdfpath(name))
plt.savefig(pngpath(name))
except Exception as e:
print 'Error while drawing sol in {0}: {1}'.format(changeName, e)
traceback.print_exc(file=sys.stdout)
finally:
plt.close()
```
%% Cell type:code id: tags:
``` python
def draw_comp_sol(changeName, params):
try:
name = 'comp_sol_{}'.format(changeName)
indices = [3, 12, 14, 19]
for i in indices:
p = params[i]
x = np.array(xrange(0,len(p[p_sol_glpk]))) # start at zero
line_javaglpk = plt.plot(x, p[p_sol_java_glpk][0]*np.ones(len(p[p_sol_glpk])), ls = linestyles[0],
c = p[p_col_def], label = p[p_label])
line_glpk = plt.plot(x, es(p[p_sol_glpk],x), ls = p[p_line_def], c = p[p_col_def])
line_gurobi = plt.plot(x, es(p[p_sol_gurobi],x), ls = p[p_line_def], c = p[p_col_def])
plt.legend(loc = 'right')
plt.ylabel('seconds')
plt.suptitle('ILP Solving Time - Comparison', fontsize = 16)
plt.savefig(pdfpath(name))
plt.savefig(pngpath(name))
except Exception as e:
print 'Error while drawing comp-sol in {0}: {1}'.format(changeName, e)
traceback.print_exc(file=sys.stdout)
finally:
plt.close()
```
%% Cell type:code id: tags:
``` python
## Currently disabled due to fragile and unused output
if False:
START_STEP, MAX_PLOTS_IN_ONE = 0, 7
for change in kinds['changes']:
print 'Change = {}'.format(change)
for strategy in kinds['strategies']:
ax_nr = 0
print 'Stategy = {}'.format(strategy)
current_plot, ax_nr, last_res = 0, 0, -1
gen_params = []
for i in xrange(len(specs)):
current_res = specs[i][2]
current_plot += 1
if current_plot > MAX_PLOTS_IN_ONE and last_res != current_res:
ax_nr += 1
current_plot = 0
gen_params.append([ax_nr, line_def[i], color_def[i], '{2:d} x ({4}*{5}*{6})'.format(*specs[i]),
safe(racket_dats[change][strategy],i,START_STEP),
safe(larceny_dats[change][strategy],i,START_STEP),
safe(java_dats[change][strategy],i)])
last_res = current_res
draw_gen(change, strategy, gen_params)
# end of for strategies
sol_params = []
ax_nr = 0
for i in xrange(len(specs)):
current_res = specs[i][2]
current_plot += 1
if current_plot > MAX_PLOTS_IN_ONE and last_res != current_res:
ax_nr += 1
current_plot = 0
sol_params.append([ax_nr, line_def[i], color_def[i], '{2:d} x ({4}*{5}*{6})'.format(*specs[i]),
safe(glpk_dats[change],i),
safe(gurobi_dats[change],i),
safe(java_glpk_dats[change],i)])
draw_sol(change, sol_params)
draw_comp_sol(change, sol_params)
```
%% Cell type:markdown id: tags:
## Settings
%% Cell type:code id: tags:
``` python
llong, lshort, lveryshort = 0, 1, 2
l = ['Flushed', '$fs$', 'f']
stt = {'normal' : ['Incremental', '$inc$', 'i'],
'flush' : l, 'flushed': l,
'noncached': ['Noncached', '$nc$', 'n']}
def strategy_to_titel(strategy, length = llong):
try:
return stt[strategy.lower()][length]
except KeyError:
raise Exception('Unknown strategy/length: {0} {1}'.format(strategy, length))
def strategy2_to_titel(ratio):
s1, s2 = ratio.split('To')
return r'${0} \rightarrow {1}$'.format(strategy_to_titel(s1, lshort)[1:-1], strategy_to_titel(s2, lshort)[1:-1])
def change_to_title(change):
# if change == 'complex':
# change = 'mixed'
return change.title()
```
%% Cell type:code id: tags:
``` python
box_facecolor, median_color, box_edgecolor, whisker_color, average_color = range(5)
def setBoxColors(bp, colors, apply_to = None, linewidth=2):
for box in bp['boxes']:
box.set(facecolor = colors[box_facecolor], linewidth=linewidth, edgecolor= colors[box_edgecolor])
for median in bp['medians']:
median.set(color=colors[median_color], linewidth=linewidth)
for whisker in bp['whiskers']:
whisker.set(ls = 'dotted', color=colors[whisker_color])
# strategy: (box.facecolor, median.color, box.edgecolor, whisker.color)
strategy_colors = {'noncached':('yellow','black','black', 'black', 'black'),
'flush': ('cyan','black','black', 'black', 'black'),
'normal': ('blue', 'white', 'blue', 'black', 'black')}
strategy_colors.update({k:('black', 'red') for k in ('NormalToFlushed', 'NormalToNoncached', 'FlushedToNoncached')})
def draw_average_line(y, c):
plt.axhline(y, ls = '-', color = c, zorder = 1, linewidth=2)
```
%% Cell type:code id: tags:
``` python
for change in kinds['changes']:
for strategy in kinds['strategies']:
cdat = racket_dats[change][strategy]
print change, strategy, np.average(cdat)
```
%% Output
update normal 1.77210853495
update flush 10.7880389785
update noncached 9.96998723118
sw normal 9.44708971546
sw flush 11.3443110599
sw noncached 13.7821516129
res normal 4.0042525181
res flush 7.77624654378
res noncached 9.03863824885
complex normal 2.04215702479
complex flush 4.21195867769
complex noncached 5.55057024793
mixed normal 3.41092375
mixed flush 22.4315775
mixed noncached 27.787
%% Cell type:markdown id: tags:
## Boxplots for generation times
%% Cell type:code id: tags:
``` python
for draw_average in (True, False):
print 'Draw_average = ', draw_average
for (data, name) in ((java_dats, 'java'), (racket_dats, 'racket'), (larceny_dats, 'larceny')):
sys.stdout.write('\n{}:'.format(name))
# change, strategy = 'mixed', 'normal'
# if True:
for change in kinds['changes']:
if change == 'complex':
continue
sys.stdout.write('\n {}'.format(change))
# if True:
for strategy in kinds['strategies']:
sys.stdout.write(','+strategy)
cdat = data[change][strategy]
if cdat.size == 1:
sys.stdout.write('/')
continue
colors = strategy_colors[strategy]
if draw_average:
draw_average_line(np.average(cdat),colors[average_color])
bp = plt.boxplot(cdat.transpose(), patch_artist=True)
setBoxColors(bp, colors, linewidth=1)
axes = plt.gca()
axes.set_ylim([0,50])
# plt.title('{0} {1} {2}'.format(name.title(), change_to_title(change), strategy_to_titel(strategy)))
plt.ylabel('seconds')
plt.xlabel('step')
plt.ylabel('seconds', fontsize = 14)
plt.xlabel('step', fontsize = 14)
plt.setp(axes.xaxis.get_majorticklabels(), fontsize = 16)
plt.setp(axes.yaxis.get_majorticklabels(), fontsize = 16)
if change == 'mixed':
for i, label in enumerate(axes.get_xticklabels()):
label.set_visible(i==0 or (i+1) % 10 == 0)
# plt.gca().tick_params(axis='x', direction='out', top = 'off', bottom = 'on')
plt.gca().tick_params(axis='x', top = 'off', bottom = 'off')
fname = 'gen_bp_{0}{1}_{2}_{3}'.format(name,'_avg' if draw_average else '',change,strategy)
plt.savefig(pdfpath(fname))
plt.savefig(pngpath(fname))
plt.close()
print ''
```
%% Output
Draw_average = True
java:
update,normal,flush/,noncached/
sw,normal/,flush/,noncached/
res,normal/,flush/,noncached/
mixed,normal/,flush/,noncached/
racket:
update,normal,flush,noncached
sw,normal,flush,noncached
res,normal,flush,noncached
mixed,normal,flush,noncached
larceny:
update,normal,flush/,noncached/
sw,normal,flush/,noncached/
res,normal,flush/,noncached/
mixed,normal/,flush/,noncached/
Draw_average = False
java:
update,normal,flush/,noncached/
sw,normal/,flush/,noncached/
res,normal/,flush/,noncached/
mixed,normal/,flush/,noncached/
racket:
update,normal,flush,noncached
sw,normal,flush,noncached
res,normal,flush,noncached
mixed,normal,flush,noncached
larceny:
update,normal,flush/,noncached/
sw,normal,flush/,noncached/
res,normal,flush/,noncached/
mixed,normal/,flush/,noncached/
%% Cell type:markdown id: tags:
## Boxplots for solving times
%% Cell type:code id: tags:
``` python
#for (data, name) in ((java_glpk_dats, 'java'), (glpk_dats, 'scheme-glpk'), (gurobi_dats, 'glpk-gurobi')):
## Skipped for now (Problems with reading solving times)
if False:
sys.stdout.write('\n{}:'.format(name))
# change = 'mixed'
# if True:
for change in kinds['changes']:
if change == 'complex':
continue
sys.stdout.write('\n {}'.format(change))
cdat = data[change]
bp = plt.boxplot(cdat.transpose(), patch_artist=True)
setBoxColors(bp, 'black', linewidth=1)
axes = plt.gca()
axes.set_ylim([0,50])
# plt.title('{0} {1} {2}'.format(name.title(), change_to_title(change), strategy_to_titel(strategy)))
plt.ylabel('seconds')
plt.xlabel('step')
if change == 'mixed':
for i, label in enumerate(axes.get_xticklabels()):
label.set_visible(i==0 or (i+1) % 10 == 0)
# plt.gca().tick_params(axis='x', direction='out', top = 'off', bottom = 'on')
plt.gca().tick_params(axis='x', top = 'off', bottom = 'off')
fname = 'sol_bp_{0}_{1}'.format(name,change)
plt.savefig(pdfpath(fname))
plt.savefig(pngpath(fname))
plt.close()
print ''
```
%% Cell type:markdown id: tags:
### Single diagram for update, res, sw. and only for racket.
%% Cell type:code id: tags:
``` python
width = 0.6
name = 'racket'
for draw_average in (True, False):
print 'Draw_average = ', draw_average
for change in ['update', 'res', 'sw']:#, 'mixed']:
#change = 'mixed'
#if True:
sys.stdout.write('\n {}'.format(change))
invisibles = []
if draw_average:
# draw horizontal lines first
for strategy in ['noncached', 'flush', 'normal']:
draw_average_line(np.average(racket_dats[change][strategy]), strategy_colors[strategy][average_color])
# now draw boxplots in front
for i, strategy in enumerate(['noncached', 'flush', 'normal']):
y = racket_dats[change][strategy]
colors = strategy_colors[strategy]
bp = plt.boxplot(y.transpose(), patch_artist=True, positions = np.arange(y.shape[0])*4+i)
setBoxColors(bp, colors)
# line, = plt.plot([1,1],c[0]+'-')
invisibles.append(mpatches.Patch(color=colors[box_facecolor], label=strategy_to_titel(strategy)))
invisibles.append(mpatches.Patch(color=colors[box_facecolor], label=strategy_to_titel(strategy), linewidth = 1))
axes = plt.gca()
y_len = y.shape[0]
axes.set_ylim([0,50])
plt.ylabel('seconds')
plt.xlabel('step')
plt.ylabel('seconds', fontsize = 14)
plt.xlabel('step', fontsize = 14)
if change == 'mixed':
axes.set_xlim([-1,y_len*4+1])
axes.set_xticklabels(range(0,111,10))
axes.set_xticks(range(1,y_len*4+41,40))
else:
axes.set_xlim([-1,y_len*4-1])
axes.set_xticklabels(range(1,y_len+1))
axes.set_xticks(range(1,y_len*4,4))
plt.setp(axes.xaxis.get_majorticklabels(), fontsize = 16)
plt.setp(axes.yaxis.get_majorticklabels(), fontsize = 16)
plt.legend(handles=invisibles, loc='best' if change == 'mixed' else 'upper center', ncol = 3, mode = "expand")
plt.legend(handles=invisibles, loc='best' if change == 'mixed' else 'upper center',
ncol = 3, mode = "expand", prop={'size':13}, handletextpad = 0.3)
fname = 'gen_bp_{0}{1}_{2}'.format(name,'_avg' if draw_average else '',change)
plt.savefig(pdfpath(fname))
plt.savefig(pngpath(fname))
plt.close()
print ''
```
%% Output
Draw_average = True
update
res
sw
Draw_average = False
update
res
sw
%% Cell type:markdown id: tags:
## Boxplot for solving times
%% Cell type:code id: tags:
``` python
#for (data, name) in ((java_glpk_dats, 'java'), (glpk_dats, 'glpk'), (gurobi_dats, 'gurobi')):
## Skipped for now (Problems with reading solving times)
if False:
print name+':'
#change = 'update'
for change in kinds['changes']:
#if True:
sys.stdout.write(change+' ')
# if True:
cdat = data[change]
plt.boxplot(cdat.transpose())
# plt.title('{0} {1}'.format(name.title(), change_to_title(change)))
plt.ylabel('seconds')
plt.xlabel('step')
fname = 'sol_bp_{0}_{1}'.format(name,change)
plt.savefig(pdfpath(fname))
plt.savefig(pngpath(fname))
plt.close()
print ''
```
%% Cell type:markdown id: tags:
## Attribute metrics
- X = {normal, flushed, noncached}
- (X1,X2) = {(normal, flushed), (normal, noncached), (flushed, noncached)}
%% Cell type:code id: tags:
``` python
def read_att_result(name = 'profiling/att-percentages.csv', adjust_number = True):
def cdir(text):
prefix = text[:-4]
number = int(text[-3:])
if not adjust_number or prefix == 'update':
return number
elif prefix == 'sw':
return 31+number
elif prefix == 'res':
return 62+number
elif prefix == 'complex':
return 93+number
elif prefix == 'mixed':
return 104+number
elif prefix == '':
pass
else:
print 'Unknown prefix "{0}"'.format(prefix)
dat = np.genfromtxt(name, delimiter=',', names=True,
#dtype=(int, float, float, float, float, float, float, float),
converters={'dir': cdir})
dat.sort(axis=0)
print 'Loaded {0} ({1} zero-values) attribute metric run(s) from {2}'.format(dat.size,
reduce(lambda total,row: total + 1*(row == 0.0) if isinstance(row,int) or isinstance(row,float) else
reduce(lambda x,y: x+1 if y == 0 else x, row, 0),dat.tolist(),0), name)
return dat
```
%% Cell type:code id: tags:
``` python
att_dat = read_att_result()
att_dat.dtype
```
%% Output
Loaded 106 (0 zero-values) attribute metric run(s) from profiling/att-percentages.csv
dtype([('dir', '<f8'), ('normalBaseline', '<f8'), ('flushedBaseline', '<f8'), ('noncachedBaseline', '<f8'), ('ratioNormalToFlushed', '<f8'), ('ratioNormalToNoncached', '<f8'), ('ratioFlushedToNoncached', '<f8'), ('speedupNormalToFlushed', '<f8'), ('speedupNormalToNoncached', '<f8'), ('speedupFlushedToNoncached', '<f8')])
%% Cell type:markdown id: tags:
1) {X}cache-miss-rate: `total.X.computed / total.X.called`
- the rate for cache-misses (1 - cache-hits) of the method X
%% Cell type:code id: tags:
``` python
width = 0.5
x = att_dat['dir']
for n, (l, c) in enumerate((('noncached', 'grey'), ('flushed', 'g'), ('normal', 'b'))):
y = att_dat[l + 'Baseline']
#plt.plot(x, y, label = l, color = c)
plt.bar(x*4*width + n*width, y, width, label = strategy_to_titel(l), color = c)
plt.legend(loc = 'best')
plt.ylabel('% computed')
#plt.suptitle('Cache Miss-Rate', fontsize = 16)
fname = 'att_miss'
plt.savefig(pdfpath(fname))
plt.savefig(pngpath(fname))
#!pdfcrop doc/att_miss.pdf doc/att_miss_cropped.pdf > /dev/null
plt.close()
```
%% Cell type:markdown id: tags:
2) ratio{X1}To{X2}: `total.{X1}.computed / total.{X2}.called`
- the efficiency of the incremental approach in comparison to the method X2,
i.e. the ratio between actual work done in X1 compared to possible work done with method X2
%% Cell type:code id: tags:
``` python
x = att_dat['dir']
for n, (name, c, ls) in enumerate((('FlushedToNoncached', 'orange', '--'),
('NormalToFlushed', 'purple', '-'),
('NormalToNoncached', 'r', '-.'))):
y = att_dat['ratio' + name]
plt.plot(x, y, label = strategy2_to_titel(name), c = c, ls = ls)
#plt.bar(x*3*width + n*width, y, width, label = l, color = c)
plt.legend(loc = 'best')
plt.ylabel('%')
#plt.suptitle('Attribute Ratios', fontsize = 16)
plt.savefig(pdfpath('att_r'))
plt.savefig(pngpath('att_r'))
#!pdfcrop doc/att_r.pdf doc/att_r_cropped.pdf > /dev/null
plt.close()
```
%% Cell type:markdown id: tags:
3) speedup{X1}To{X2}: `(total.{X2}.computed / total.{X2}.called) - (total.{X1}.computed / total.{X2}.called)`
- = `baseline({X2}) - ratio({X1}, {X2})`
- the "speed-up" of the incremental approach (normal or flushed) in comparison to the method X2
%% Cell type:code id: tags:
``` python
x = att_dat['dir']
for n, (l, c) in enumerate((('NormalToNoncached', 'r'),
('FlushedToNoncached', 'orange'),
('NormalToFlushed', 'yellow'))):
y = att_dat['speedup' + l]
plt.plot(x, y, label = strategy2_to_titel(l), color = c)
#plt.bar(x*3*width + n*width, y, width, label = l, color = c)
plt.legend(loc = 'best')
plt.ylabel('%')
plt.suptitle('Attribute Speed-Ups', fontsize = 16)
plt.savefig(pdfpath('att_sp'))
plt.savefig(pngpath('att_sp'))
#!pdfcrop doc/att_sp.pdf doc/att_sp_cropped.pdf > /dev/null
plt.close()
```
%% Cell type:markdown id: tags:
## Boxplots for attribute measures
%% Cell type:code id: tags:
``` python
att_totals = {change: read_att_result(name = 'profiling/splitted/att-percentages_{}.csv'.format(change),
adjust_number = False) for change in kinds['changes'] if change != 'complex'}
```
%% Output
Loaded 31 (0 zero-values) attribute metric run(s) from profiling/splitted/att-percentages_update.csv
Loaded 31 (0 zero-values) attribute metric run(s) from profiling/splitted/att-percentages_sw.csv
Loaded 31 (0 zero-values) attribute metric run(s) from profiling/splitted/att-percentages_res.csv
Loaded 2 (0 zero-values) attribute metric run(s) from profiling/splitted/att-percentages_mixed.csv
%% Cell type:code id: tags:
``` python
PRINT_NONCACHED = True
```
%% Cell type:code id: tags:
``` python
def draw_att(x, fcolumn, flabel, title, fname_part, rotation_xaxis = None):
""" Draw boxplots for attribute
@param:fcolum: function with one parameter (name) returning column name
@param:flabel: function with one parameter (name) returning label
@param:title: title of plot
@param:fname_part: unique filename part for storing plot
@param:rotation_xaxis: rotation of xaxis ticks (default: no rotation)"""
try:
changes_to_draw = [c for c in kinds['changes'] if c != 'complex']
fig, axes = plt.subplots(ncols=len(changes_to_draw))
# if not rotation_xaxis:
# print dir(fig)
# fig.suptitle(title, fontsize = 16)
l = ['normal', 'flushed']
if PRINT_NONCACHED:
l.append('noncached')
for i, change in enumerate(changes_to_draw):
att_dat = att_totals[change]
axes[i].boxplot([att_dat[fcolumn(name)] for name in x],
labels = [flabel(name) for name in x])
# axes[i].set_title(change_to_title(change), fontsize=10)
if i > 0:
axes[i].set_yticklabels([])
for ax in axes.flatten():
if PRINT_NONCACHED:
ax.set_ylim([0,1.05])
if rotation_xaxis:
plt.setp(ax.xaxis.get_majorticklabels(), rotation=rotation_xaxis)
plt.setp(ax.xaxis.get_majorticklabels(), fontsize = 15)
plt.setp(ax.yaxis.get_majorticklabels(), fontsize = 15)
# plt.axes().xaxis.set_major_locator(matplotlib.ticker.FixedLocator(range(0, 3, 2)))
# plt.axes().xaxis.set_minor_locator(matplotlib.ticker.FixedLocator([1]))
# plt.axes().xaxis.set_minor_formatter(matplotlib.ticker.FormatStrFormatter("%s"))
# plt.axes().tick_params(which='major', pad=20, axis='x')
plt.subplots_adjust(top=0.85)
plt.savefig(pdfpath(fname_part))
plt.savefig(pngpath(fname_part))
# !pdfcrop {'doc/{}.pdf'.format(fname_part)} {'doc/{}_cropped.pdf'.format(fname_part)} > /dev/null
except Exception as e:
print 'Error while drawing att in {0}: {1}'.format(change, e)
traceback.print_exc(file=sys.stdout)
finally:
plt.close()
```
%% Cell type:code id: tags:
``` python
def baseline(x):
return '{}Baseline'.format('flushed' if x == 'flush' else x)
print 'Drawing cache-miss rate'
draw_att(kinds['strategies'], baseline, functools.partial(strategy_to_titel, length = lshort),
'Attribute Cache-Miss Rate', 'att_box_miss')
x = ['NormalToFlushed', 'NormalToNoncached', 'FlushedToNoncached']
print 'Drawing ratios'
draw_att(x, lambda x: 'ratio{}'.format(x), strategy2_to_titel, 'Attribute Ratios', 'att_box_r', rotation_xaxis = 30)
print 'Drawing speed-ups'
draw_att(x, lambda x: 'speedup{}'.format(x), strategy2_to_titel, 'Attribute Speedups', 'att_box_sp', rotation_xaxis = 30)
```
%% Output
Drawing cache-miss rate
Drawing ratios
Drawing speed-ups
%% Cell type:markdown id: tags:
## Graph for attribute metrics over steps
%% Cell type:code id: tags:
``` python
def draw_att_graph(kind, nr,log=False):
f_pattern = 'profiling/splitted/att-totals-{0}-{1}-{2}.csv'
cs = ['b',(0,0,0.5),'g',(0,0.5,0),'r',(0.7,0,0)]
def cv_name(x):
return int(x[-3:])
try:
dats = [(s,np.genfromtxt(f_pattern.format(kind,nr,s), delimiter=',', names=True,
converters={'name': cv_name})) for s in kinds['strategies']]
for i,(label,dat) in enumerate(dats):
plt.plot(dat['comp'],color=cs[2*i], label = label)
plt.plot(dat['called'], ls=':',color=cs[2*i+1])
if log:
ax = plt.gca()
ax.set_yscale('log')
plt.legend(loc='best')
fname_part = '{0}_{1}{2}'.format(kind,nr,'_log' if log else '')
plt.savefig(pdfpath(fname_part))
plt.savefig(pngpath(fname_part))
# !pdfcrop {'doc/{}.pdf'.format(fname_part)} {'doc/{}_cropped.pdf'.format(fname_part)} > /dev/null
except Exception as e:
print 'Error while drawing att graph for {0}: {1}'.format(kind, e)
traceback.print_exc(file=sys.stdout)
finally:
plt.close()
```
%% Cell type:code id: tags:
``` python
draw_att_graph('mixed', '001')
draw_att_graph('mixed', '001', log = True)
```
%% Cell type:markdown id: tags:
## Graph for memory measurement
%% Cell type:code id: tags:
``` python
def draw_mem_graph(kind, nr):
global dat_mem
f = 'profiling/{0}-{1}/memory.csv'.format(kind, nr)
print f
def cv_date(x):
# return datetime.strptime(x, '%Y-%m-%dT%H:%M:%S.%f')
return (datetime.strptime(x, '%Y-%m-%dT%H:%M:%S.%f') - datetime(1970,1,1)).total_seconds()
def cv_mem(x):
return int(x) / 1024
raw_data = []
with open(f) as fd:
next(fd)
for l in fd:
toks = l.split(',')
raw_data.append((cv_date(toks[0]), cv_mem(toks[1])))
# dat = np.genfromtxt(f, delimiter=',', names=True, dtype = ('datetime64[us]', int),
# converters={'timestamp': cv_date, 'vmsize':cv_mem})
## print raw_data
dat_mem = np.array(raw_data, dtype = [('timestamp', int), ('vmsize', int)])
# dat_mem.dtype.names = 'timestamp', 'vmsize'
try:
plt.plot(dat_mem['vmsize'])
plt.ylabel('VmSize (MB)')
plt.xlabel('Time (Seconds)')
# plt.legend(loc='best')
fname_part = 'memory_{0}-{1}'.format(kind,nr)
plt.savefig(pdfpath(fname_part))
plt.savefig(pngpath(fname_part))
# !pdfcrop {'doc/{}.pdf'.format(fname_part)} {'doc/{}_cropped.pdf'.format(fname_part)} > /dev/null
except Exception as e:
print 'Error while drawing memory graph for {0}: {1}'.format(kind, e)
traceback.print_exc(file=sys.stdout)
finally:
plt.close()
```
%% Cell type:code id: tags:
``` python
draw_mem_graph('mixed', '001')
draw_mem_graph('mixed', '002')
```
%% Output
profiling/mixed-001/memory.csv
profiling/mixed-002/memory.csv
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment