From 15bd11a9f48b995213d15fa2f54bd43b72ae4023 Mon Sep 17 00:00:00 2001
From: rschoene <rene.schoene@tu-dresden.de>
Date: Thu, 22 Oct 2015 15:16:15 +0200
Subject: [PATCH] WIP: getting measurement plots right again for change kinds
---
Makefile | 1 +
ilp-measurement.ipynb | 245 ++++++++++++++++++++++++++++++++----------
2 files changed, 189 insertions(+), 57 deletions(-)
diff --git a/Makefile b/Makefile
index 04234fb..94752fb 100644
--- a/Makefile
+++ b/Makefile
@@ -10,6 +10,7 @@ LDEPS := $(shell sed -e '1,/@sources:/d' -e '/^\#/d' dependencies.txt | while re
all: racket
racket: $(RDEPS) Makefile
+ @rm -f $(RACKET_BUILD_DIR)/mquat/ilp.ss
larceny: $(LSRC)
@mkdir -p $(LARCENY_BUILD_DIR)
diff --git a/ilp-measurement.ipynb b/ilp-measurement.ipynb
index 12117b9..230d618 100644
--- a/ilp-measurement.ipynb
+++ b/ilp-measurement.ipynb
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 21,
+ "execution_count": 1,
"metadata": {
"collapsed": false
},
@@ -17,7 +17,7 @@
},
{
"cell_type": "code",
- "execution_count": 39,
+ "execution_count": 2,
"metadata": {
"collapsed": false
},
@@ -130,6 +130,27 @@
" return not type(a) is np.ndarray"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": 33,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Kinds: {u'strategies': [u'normal', u'flush', u'noncached'], u'changes': [u'update', u'sw', u'res']}\n"
+ ]
+ }
+ ],
+ "source": [
+ "with open('profiling/kinds.json') as fd:\n",
+ " kinds = json.load(fd)\n",
+ "print 'Kinds: {}'.format(kinds)"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -139,25 +160,53 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 58,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
- "def read_results(prefix, name, dtype, data_column, since):\n",
- " f = 'profiling/{0}-{1}-results.csv'.format(prefix, name)\n",
- " def mkdate(text):\n",
+ "no_split, split_change_only, split_both = 0, 1, 2\n",
+ "def read_single_result(f, name, dtype, data_column, since):\n",
+ " def convdate(text):\n",
" return datetime.strptime(text, '%Y-%m-%dT%H:%M:%S.%f')\n",
- " def mkstep(text):\n",
+ " def convdir(text):\n",
+ " return int(text[-3:])\n",
+ " def convstep(text):\n",
" return int(text[0:2])\n",
- " dat = np.genfromtxt(f, delimiter=',', names=True, dtype=dtype, converters={'timestamp':mkdate, 'step':mkstep})\n",
+ " def safe_div(a, b):\n",
+ " return a/b if b>0 else \"{}!\".format(a)\n",
+ " dat = np.genfromtxt(f, delimiter=',', names=True, dtype=dtype,\n",
+ " converters={'timestamp':convdate, 'step':convstep, 'dir': convdir})\n",
" if since:\n",
" dat = dat[dat['timestamp'] > since ]\n",
" dat2 = get_average_times(dat, 1, 2, data_column).transpose()\n",
- " print 'Loaded {0} records for {1} ({2[0]}x{2[1]} unique) ~= {3} run(s)'.format(\n",
- " len(dat), name, dat2.shape, len(dat)/dat2.size)\n",
- " return dat2"
+ " if dat2.size == 0:\n",
+ " print 'Did not load any record for {}'.format(name)\n",
+ " else:\n",
+ " print 'Loaded {0} records for {1} ({2[0]}x{2[1]} unique) ~= {3} run(s)'.format(\n",
+ " len(dat), name, dat2.shape, safe_div(len(dat),dat2.size))\n",
+ " return dat2\n",
+ "\n",
+ "def read_results(prefix, name, dtype, data_column, since, splitted = no_split):\n",
+ " if splitted in (split_change_only, split_both):\n",
+ " result = {}\n",
+ " for change in kinds['changes']:\n",
+ " result.setdefault(change, {})\n",
+ " if splitted == split_both:\n",
+ " for strategy in kinds['strategies']:\n",
+ " result[change].setdefault(strategy, {})\n",
+ " new_name = '{0}_{1}_{2}'.format(change, strategy, name)\n",
+ " f = 'profiling/splitted/{0}_{1}.csv'.format(prefix, new_name)\n",
+ " result[change][strategy] = read_single_result(f, new_name, dtype, data_column, since)\n",
+ " else: # splitted == split_change_only\n",
+ " new_name = '{0}_{1}'.format(change, name)\n",
+ " f = 'profiling/splitted/{0}_{1}.csv'.format(prefix, new_name)\n",
+ " result[change] = read_single_result(f, new_name, dtype, data_column, since)\n",
+ " return result\n",
+ " else: # splitted = no_split\n",
+ " f = 'profiling/{0}-{1}-results.csv'.format(prefix, name)\n",
+ " return read_single_result(f, name, dtype, data_column, since)"
]
},
{
@@ -169,16 +218,17 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 59,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"def read_gen_results(impl, since = None):\n",
- " return read_results('gen', impl, ('datetime64[us]', int, int, float), 3, since)\n",
+ " return read_results('gen', impl, ('datetime64[us]', int, int, float), 3, since, splitted = split_both)\n",
"def read_sol_results(solver, since = None):\n",
- " return read_result('sol', solver, ('datetime64[us]', int, int, int, int, int, float, float), 7, since)\n",
+ " return read_results('sol', solver, ('datetime64[us]', int, int, int, int, int, float, float), 7, since,\n",
+ " splitted = split_change_only)\n",
"def read_att_result(name = 'profiling/att-percentages.csv'):\n",
" def cdir(text):\n",
" prefix = text[:-4]\n",
@@ -245,7 +295,7 @@
},
{
"cell_type": "code",
- "execution_count": 83,
+ "execution_count": 60,
"metadata": {
"collapsed": false,
"scrolled": true
@@ -255,10 +305,33 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "Loaded 2124 records for plt-r6rs (8x32 unique) = 8 run(s)\n",
- "Loaded 264 records for larceny (8x31 unique) = 1 run(s)\n",
- "Loaded 88 records for java (1x31 unique) = 2 run(s)\n",
- "Loaded 39 attribute metrics\n"
+ "Loaded 328 records for update_normal_plt-r6rs (8x31 unique) ~= 1 run(s)\n",
+ "Loaded 256 records for update_flush_plt-r6rs (8x31 unique) ~= 1 run(s)\n",
+ "Loaded 496 records for update_noncached_plt-r6rs (8x31 unique) ~= 2 run(s)\n",
+ "Loaded 63 records for sw_normal_plt-r6rs (7x4 unique) ~= 2 run(s)\n",
+ "Loaded 28 records for sw_flush_plt-r6rs (7x4 unique) ~= 1 run(s)\n",
+ "Loaded 140 records for sw_noncached_plt-r6rs (7x13 unique) ~= 1 run(s)\n",
+ "Loaded 63 records for res_normal_plt-r6rs (7x4 unique) ~= 2 run(s)\n",
+ "Loaded 28 records for res_flush_plt-r6rs (7x4 unique) ~= 1 run(s)\n",
+ "Loaded 147 records for res_noncached_plt-r6rs (7x14 unique) ~= 1 run(s)\n",
+ "Did not load any record for update_normal_larceny\n",
+ "Did not load any record for update_flush_larceny\n",
+ "Did not load any record for update_noncached_larceny\n",
+ "Did not load any record for sw_normal_larceny\n",
+ "Did not load any record for sw_flush_larceny\n",
+ "Did not load any record for sw_noncached_larceny\n",
+ "Did not load any record for res_normal_larceny\n",
+ "Did not load any record for res_flush_larceny\n",
+ "Did not load any record for res_noncached_larceny\n",
+ "Did not load any record for update_normal_java\n",
+ "Did not load any record for update_flush_java\n",
+ "Did not load any record for update_noncached_java\n",
+ "Did not load any record for sw_normal_java\n",
+ "Did not load any record for sw_flush_java\n",
+ "Did not load any record for sw_noncached_java\n",
+ "Did not load any record for res_normal_java\n",
+ "Did not load any record for res_flush_java\n",
+ "Did not load any record for res_noncached_java\n"
]
}
],
@@ -270,7 +343,7 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 61,
"metadata": {
"collapsed": false,
"scrolled": true
@@ -280,9 +353,15 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "Loaded 23 records for java (1x23 unique) = 1 run(s)\n",
- "Loaded 1656 records for glpk (8x23 unique) = 9 run(s)\n",
- "Loaded 216 records for gurobi (8x27 unique) = 1 run(s)\n"
+ "Loaded 23 records for update_java (1x23 unique) ~= 1 run(s)\n",
+ "Did not load any record for sw_java\n",
+ "Did not load any record for res_java\n",
+ "Loaded 1656 records for update_glpk (8x23 unique) ~= 9 run(s)\n",
+ "Did not load any record for sw_glpk\n",
+ "Did not load any record for res_glpk\n",
+ "Did not load any record for update_gurobi\n",
+ "Did not load any record for sw_gurobi\n",
+ "Did not load any record for res_gurobi\n"
]
}
],
@@ -294,7 +373,20 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 63,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "p_ax_nr, p_line_def, p_col_def, p_label = 0, 1, 2, 3\n",
+ "p_gen_racket, p_gen_larceny, p_gen_java = 4, 5, 6\n",
+ "p_sol_glpk, p_sol_gurobi, p_sol_java_glpk = 4, 5, 6"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 64,
"metadata": {
"collapsed": false,
"scrolled": false
@@ -313,7 +405,7 @@
"\n",
" lines, labels = [], []\n",
" for p in params:\n",
- " ax_tup = ax_arr if one_plot else ax_arr[p[0]]\n",
+ " ax_tup = ax_arr if one_plot else ax_arr[p[p_ax_nr]]\n",
" ax_j = ax_tup[0]\n",
" ax_r = ax_tup[1]\n",
" ax_l = ax_tup[2]\n",
@@ -321,13 +413,14 @@
" # ax_r.set_ylim([0,10])\n",
" # ax_l.set_ylim([0,10])\n",
" # x_g = np.array(xrange(1,len(p[1])+1)) # start at one, since first gen-time is cut\n",
- " x_g = np.array(xrange(START_STEP,len(p[1])+START_STEP)) # start at zero\n",
- " line_java = ax_j.plot(x_g, p[8][0]*np.ones(len(p[1])), ls = p[5], c = p[6], label = p[7])\n",
- " line_racket = ax_r.plot(x_g, p[1], ls = p[5], c = p[6], label = p[7])\n",
- " line_larceny = ax_l.plot(x_g, p[2], ls = p[5], c = p[6], label = p[7])\n",
+ " x_g = np.array(xrange(START_STEP,len(p[p_gen_racket])+START_STEP)) # start at zero\n",
+ " line_java = ax_j.plot(x_g, p[p_gen_java][0]*np.ones(len(p[p_gen_racket])), ls = p[p_line_def],\n",
+ " c = p[p_col_def], label = p[p_label])\n",
+ " line_racket = ax_r.plot(x_g, p[p_gen_racket], ls = p[p_line_def], c = p[p_col_def], label = p[p_label])\n",
+ " line_larceny = ax_l.plot(x_g, p[p_gen_larceny], ls = p[p_line_def], c = p[p_col_def], label = p[p_label])\n",
" ax_l.legend(loc='upper left', bbox_to_anchor=(1, 1.02))\n",
" lines.append(line_racket[0])\n",
- " labels.append(p[7])\n",
+ " labels.append(p[p_label])\n",
"\n",
" for ax in ax_arr if one_plot else ax_arr.flatten():\n",
" ax.set_ylabel('seconds')\n",
@@ -352,7 +445,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 65,
"metadata": {
"collapsed": false
},
@@ -369,17 +462,18 @@
"\n",
" lines, labels = [], []\n",
" for p in params:\n",
- " ax_tup = ax_arr if one_plot else ax_arr[p[0]]\n",
+ " ax_tup = ax_arr if one_plot else ax_arr[p[p_ax_nr]]\n",
" ax_javaglpk = ax_tup[0]\n",
" ax_glpk = ax_tup[1]\n",
" ax_gurobi = ax_tup[2]\n",
- " x = np.array(xrange(0,len(p[3]))) # start at zero\n",
- " line_javaglpk = ax_javaglpk.plot(x, p[9][0]*np.ones(len(p[3])), ls = p[5], c = p[6], label = p[7])\n",
- " line_glpk = ax_glpk.plot(x, p[3], ls = p[5], c = p[6], label = p[7])\n",
- " line_gurobi = ax_gurobi.plot(x, p[4], ls = p[5], c = p[6], label = p[7])\n",
+ " x = np.array(xrange(0,len(p[p_sol_glpk]))) # start at zero\n",
+ " line_javaglpk = ax_javaglpk.plot(x, p[p_sol_java_glpk][0]*np.ones(len(p[p_sol_glpk])), ls = p[p_line_def],\n",
+ " c = p[p_col_def], label = p[p_label])\n",
+ " line_glpk = ax_glpk.plot(x, p[p_sol_glpk], ls = p[p_line_def], c = p[p_col_def], label = p[p_label])\n",
+ " line_gurobi = ax_gurobi.plot(x, p[p_sol_gurobi], ls = p[p_line_def], c = p[p_col_def], label = p[p_label])\n",
" ax_gurobi.legend(loc='upper left', bbox_to_anchor=(1, 1.02))\n",
" lines.append(line_glpk[0])\n",
- " labels.append(p[7])\n",
+ " labels.append(p[p_label])\n",
"\n",
" for ax in ax_arr if one_plot else ax_arr.flatten():\n",
" ax.set_ylabel('seconds')\n",
@@ -404,7 +498,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 66,
"metadata": {
"collapsed": false,
"scrolled": true
@@ -416,9 +510,10 @@
" indices = [3, 12, 14, 19]\n",
" for i in indices:\n",
" p = params[i]\n",
- " line_javaglpk = plt.plot(x, p[9][0]*np.ones(len(p[3])), ls = linestyles[0], c = p[6], label = p[7])\n",
- " line_glpk = plt.plot(x, p[3], ls = linestyles[1], c = p[6])\n",
- " line_gurobi = plt.plot(x, p[4], ls = linestyles[3], c = p[6])\n",
+ " line_javaglpk = plt.plot(x, p[p_sol_java_glpk][0]*np.ones(len(p[p_sol_glpk])), ls = linestyles[0],\n",
+ " c = p[p_col_def], label = p[p_label])\n",
+ " line_glpk = plt.plot(x, p[p_sol_glpk], ls = p[p_line_def], c = p[p_col_def])\n",
+ " line_gurobi = plt.plot(x, p[p_sol_gurobi], ls = p[p_line_def], c = p[p_col_def])\n",
" plt.legend(loc = 'right')\n",
" plt.ylabel('seconds')\n",
" plt.suptitle('ILP Solving Time - Comparison', fontsize = 16)\n",
@@ -429,31 +524,69 @@
},
{
"cell_type": "code",
- "execution_count": 13,
+ "execution_count": 68,
"metadata": {
"collapsed": false,
"scrolled": true
},
- "outputs": [],
+ "outputs": [
+ {
+ "ename": "IndexError",
+ "evalue": "too many indices for array",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[1;31mIndexError\u001b[0m Traceback (most recent call last)",
+ "\u001b[1;32m<ipython-input-68-e8120160290e>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m 16\u001b[0m gen_params.append([ax_nr, line_def[i], color_def[i], '{2:d} x ({4}*{5}*{6})'.format(*specs[i]),\n\u001b[0;32m 17\u001b[0m \u001b[0msafe\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mracket_dats\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mchange\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mstrategy\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mSTART_STEP\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 18\u001b[1;33m \u001b[0msafe\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlarceny_dats\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mchange\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mstrategy\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mSTART_STEP\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 19\u001b[0m safe(java_dat[change][strategy],i)])\n\u001b[0;32m 20\u001b[0m \u001b[0mlast_res\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcurrent_res\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+ "\u001b[1;32m<ipython-input-10-27c199385393>\u001b[0m in \u001b[0;36msafe\u001b[1;34m(a, i, start)\u001b[0m\n\u001b[0;32m 3\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0ma\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mstart\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 4\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mIndexError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
+ "\u001b[1;31mIndexError\u001b[0m: too many indices for array"
+ ]
+ }
+ ],
"source": [
- "for change_index in xrange(len(racket_dats) + 1):\n",
- " START_STEP = 0\n",
- " MAX_PLOTS_IN_ONE, current_plot, ax_nr, last_res = 7, 0, 0, -1\n",
- " params = []\n",
+ "START_STEP, MAX_PLOTS_IN_ONE = 0, 7\n",
+ "for change in kinds['changes']:\n",
+ " for strategy in kinds['strategies']:\n",
+ " current_plot, ax_nr, last_res = 0, 0, -1\n",
+ " gen_params = []\n",
+ " for i in xrange(len(specs)):\n",
+ " current_res = specs[i][2]\n",
+ " current_plot += 1\n",
+ " if current_plot > MAX_PLOTS_IN_ONE and last_res != current_res:\n",
+ " ax_nr += 1\n",
+ " current_plot = 0\n",
+ "# params.append([ax_nr, safe(racket_dats[change][strategy],i,START_STEP), safe(larceny_dats[change][strategy],i,START_STEP),\n",
+ "# safe(glpk_dat[change][strategy],i), safe(gurobi_dat[change][strategy],i),\n",
+ "# line_def[i], color_def[i], '{2:d} x ({4}*{5}*{6})'.format(*specs[i]),\n",
+ "# safe(java_dat[change][strategy],i), safe(java_glpk_dat[change][strategy],i)])\n",
+ " gen_params.append([ax_nr, line_def[i], color_def[i], '{2:d} x ({4}*{5}*{6})'.format(*specs[i]),\n",
+ " safe(racket_dats[change][strategy],i,START_STEP),\n",
+ " safe(larceny_dats[change][strategy],i,START_STEP),\n",
+ " safe(java_dat[change][strategy],i)])\n",
+ " last_res = current_res\n",
+ " try:\n",
+ " draw_gen(gen_params)\n",
+ " except:\n",
+ " print 'Error while drawing gen in {0}-{1}'.format(change, strategy)\n",
+ " sol_params = []\n",
" for i in xrange(len(specs)):\n",
" current_res = specs[i][2]\n",
" current_plot += 1\n",
" if current_plot > MAX_PLOTS_IN_ONE and last_res != current_res:\n",
" ax_nr += 1\n",
" current_plot = 0\n",
- " params.append([ax_nr, safe(racket_dat[change_index],i,START_STEP), safe(larceny_dat[change_index],i,START_STEP),\n",
- " safe(glpk_dat[change_index],i), safe(gurobi_dat[change_index],i),\n",
- " line_def[i], color_def[i], '{2:d} x ({4}*{5}*{6})'.format(*specs[i]),\n",
- " safe(java_dat[change_index],i), safe(java_glpk_dat[change_index],i)])\n",
- " last_res = current_res\n",
- " draw_gen(params)\n",
- " draw_sol(params)\n",
- " draw_comp_sol(params)"
+ " sol_params.append([ax_nr, line_def[i], color_def[i], '{2:d} x ({4}*{5}*{6})'.format(*specs[i]),\n",
+ " safe(glpk_dat[change][strategy],i),\n",
+ " safe(gurobi_dat[change][strategy],i),\n",
+ " safe(java_glpk_dat[change][strategy],i)])\n",
+ " try:\n",
+ " draw_sol(params)\n",
+ " except:\n",
+ " print 'Error while drawing sol in {0}-{1}'.format(change, strategy)\n",
+ " try:\n",
+ " draw_comp_sol(params)\n",
+ " except:\n",
+ " print 'Error while drawing comp-sol in {0}-{1}'.format(change, strategy)\n"
]
},
{
@@ -663,8 +796,6 @@
}
],
"source": [
- "with open('profiling/kinds.json') as fd:\n",
- " d = json.load(fd)\n",
"att_totals = {}\n",
"for change in d['changes']:\n",
" att_totals[change] = read_att_result(name = 'profiling/splitted/att-percentages_{}.csv'.format(change))"
--
GitLab