From fe7e9fd831cefd39f20ef700612603f3ca21dbdd Mon Sep 17 00:00:00 2001
From: rschoene <rene.schoene@tu-dresden.de>
Date: Fri, 30 Oct 2015 09:43:50 +0100
Subject: [PATCH] New measurement case: complex
- Complex scenaria involving all three change kinds
- Added more tasks (factor and kill-lines)
- Added taget clean to Makefile
- Description of kinds of changes
---
Makefile | 6 +-
doc/kinds.md | 55 ++++++++++++
ilp-measurement.scm | 213 +++++++++++++++++++++++++-------------------
ilp_measurement.py | 36 +++++---
utils.py | 36 +++++++-
5 files changed, 238 insertions(+), 108 deletions(-)
create mode 100644 doc/kinds.md
diff --git a/Makefile b/Makefile
index 94752fb..20b0722 100644
--- a/Makefile
+++ b/Makefile
@@ -5,7 +5,7 @@ LARCENY_BUILD_DIR = larceny-bin
RDEPS := $(shell sed -e '1,/@sources:/d' -e '/^\#/d' dependencies.txt | while read l; do echo $(RACKET_BUILD_DIR)/mquat/$$l.ss; done)
LDEPS := $(shell sed -e '1,/@sources:/d' -e '/^\#/d' dependencies.txt | while read l; do echo $(LARCENY_BUILD_DIR)/mquat/$$l.sls; done)
-.PHONY: larceny racket run sockets
+.PHONY: larceny racket run sockets clean
# larceny builds everytime, so not included in default target
all: racket
@@ -32,3 +32,7 @@ sockets: SocketSend.class SocketReceive.class
ilp-noncached.scm: ilp.scm
sed 's/(lambda (n/#f (lambda (n/g' $< > $@
+
+clean:
+ rm -rf $(RACKET_BUILD_DIR)/*
+ rm -rf $(LARCENY_BUILD_DIR)/*
diff --git a/doc/kinds.md b/doc/kinds.md
new file mode 100644
index 0000000..cb9dc80
--- /dev/null
+++ b/doc/kinds.md
@@ -0,0 +1,55 @@
+## Kinds of changes
+
+### Update
+
+- init
+- two times: change of value in first resource
+- change of values in first three resources
+- change of values in all resource to low value
+- three times: change of values in all resource to random value
+
+### SW
+
+- init
+- three times: create a new mode for each implementation of each component
+- three times: remove half all available modes
+
+### Res
+
+- init
+- remove first resource from tree (1)
+- add previously (1) removed resource to tree again
+- remove half of the resources from tree (2)
+- add previously (2) removed resources to tree again
+- remove half of the resources from tree and change values of remaining resources (3)
+- add previously (3) removed resources to tree again
+
+### Complex
+
+- init (4 sw component, N containers)
+- change values of containers
+- create new modes for each implementation
+- change values of containers
+- add new component
+- change values of containers
+- add new containers
+- change values of containers
+- (change nothing)
+- remove some containers
+- remove some modes
+
+## Strategies
+
+### Normal/Incremental
+
+- normal RACR execution, cached used and untouched between steps
+
+## Flushed
+
+- normal RACR execution, cached used
+- cache is flushed after generation of the ILP for each change
+
+## Noncached
+
+- all ILP-generation related attributes are note cached
+- basic attributes (e.g. compute list of valid hardware containers)
diff --git a/ilp-measurement.scm b/ilp-measurement.scm
index a8b5d7c..8a79483 100644
--- a/ilp-measurement.scm
+++ b/ilp-measurement.scm
@@ -15,40 +15,21 @@
(define resource-test 'resource-test)
(define sw-test 'sw-test)
(define update-test 'update-test)
- (define raw-paper-params
+ (define complex-test 'complex-test)
+ (define raw-short-params
; HW = number hw. S = number of sub-per-hw. #C[omp], #I[mpl], #M[ode]
; ID HW S #C #I #M
(list (list 1 10 0 2 1 2)
- (list 2 10 0 10 1 2)
- (list 3 10 0 10 10 2)
- (list 4 10 0 10 10 10)
- (list 5 20 0 1 1 2)
- (list 6 20 0 10 1 2)
- (list 7 20 0 10 10 2)
- (list 8 20 0 10 10 10)
- (list 9 20 0 20 2 2)
- (list 10 25 0 25 2 2)
- (list 11 30 0 30 2 2)
- (list 12 40 0 40 2 2)
- (list 13 50 0 50 2 2)
- (list 14 60 0 50 2 2)
- (list 15 70 0 50 2 2)
- (list 16 80 0 50 2 2)
- (list 17 80 0 60 2 2)
- (list 18 80 0 70 2 2)
- (list 19 80 0 80 2 2)
- (list 20 90 0 60 2 2)
- (list 21 90 0 70 2 2)
- (list 22 90 0 80 2 2)
- (list 23 90 0 90 2 2)
- (list 24 100 10 10 2 2)
- (list 25 200 10 10 2 2)
- (list 26 300 10 10 2 2)
- (list 27 400 10 10 2 2)
- (list 28 10 0 10 2 2)
- (list 29 10 0 20 2 2)
- (list 30 10 0 40 2 2)
- (list 31 10 0 80 2 2)))
+ (list 2 40 0 40 2 2)
+ (list 3 50 0 50 2 2)
+ (list 4 60 0 50 2 2)
+ (list 5 70 0 50 2 2)
+ (list 6 80 0 50 2 2)
+ (list 7 90 0 60 2 2)
+ (list 8 100 10 10 2 2)
+ (list 9 200 10 10 2 2)
+ (list 10 300 10 10 2 2)
+ (list 11 400 10 10 2 2)))
(define raw-params
; HW = number hw. S = number of sub-per-hw. #C[omp], #I[mpl], #M[ode]
; ID HW S #C #I #M
@@ -85,18 +66,24 @@
(list 31 10 0 80 2 2)))
(define params
(append
- ; paper params
+ ; res and sw params
(reverse
(fold-left
(lambda (all l) (let ([rest (append (cdr l) (list (list (lambda _ #t) #f #f #f)))])
(cons* (cons* (dirname "res-" (car l)) resource-test rest)
(cons* (dirname "sw-" (car l)) sw-test rest)
+ (cons* (dirname "update-" (car l)) update-test rest)
all)))
- (list) raw-paper-params))
- ; normal params
- (map (lambda (l) (append (cons* (dirname "update-" (car l)) update-test (cdr l))
- (list (list (lambda _ #t) #f #f #f))))
- raw-params)))
+ (list) raw-params))
+ ; complex params
+ (map (lambda (l) (append (cons* (dirname "complex-" (car l)) complex-test (cdr l))
+ (list (list (lambda _ #t) #f #f #f))))
+ raw-short-params)
+; ; normal params
+; (map (lambda (l) (append (cons* (dirname "update-" (car l)) update-test (cdr l))
+; (list (list (lambda _ #t) #f #f #f))))
+; raw-params)
+ ))
(define (valf val) (lambda _ val))
@@ -189,65 +176,65 @@
(for-each (lambda (rp) (rewrite-add (cdr rp) (car rp))) (reverse removed))
(rw* rt "load" #f ast) (sit id-s "07-add-odd-pes" ast) (display+flush "."))))
- (define (run-sw-test id-s specs)
- (let* ([ast (cst id-s specs)]
- [energy (ast-find-child (lambda (i child) (string=? (->name child) pn-energy)) (->RealProperty* (->SWRoot ast)))])
- (define (add-comp comp-nr)
- (debug "#create new comp" comp-nr)
- (let ([new (:Comp mquat-spec (node-name "c" (list comp-nr)) (list) #f
- (list (:RealProperty mquat-spec (node-name "p" (list comp-nr))
- #f "runtime" 'increasing agg-sum)
- (:PropertyRef mquat-spec energy)))])
- (rewrite-add (->Comp* (->SWRoot ast)) new) new))
- (define (find-create l prefix lon make-new)
- (let ([name (node-name prefix lon)])
- (or (ast-find-child (lambda (i child) (string=? (->name child) name)) l) (make-new))))
- (define (find-create-comp comp-nr) (find-create (->Comp* (->SWRoot ast)) "c" (list comp-nr) (lambda _ (add-comp comp-nr))))
- (define (add-impl comp-nr impl-nr reqcomps)
- (debug "#create new impl" comp-nr impl-nr reqcomps)
- (let ([new (:Impl mquat-spec (node-name "i" (list impl-nr comp-nr)) (list)
- (map (lambda (nr) (find-create-comp nr)) reqcomps) #f #f)])
- (rewrite-add (->Impl* (find-create-comp comp-nr)) new) new))
- (define (find-create-impl comp-nr impl-nr reqcomps) (find-create (->Impl* (find-create-comp comp-nr)) "i"
- (list impl-nr comp-nr)
- (lambda _ (add-impl comp-nr impl-nr reqcomps))))
- (define (add-mode comp-nr impl-nr mode-nr req-comp-nr load-f energy-f prov-f prev-f)
- (debug "#create new mode" comp-nr impl-nr mode-nr req-comp-nr)
- (let* ([impl (find-create-impl comp-nr impl-nr (if req-comp-nr (list req-comp-nr) (list)))]
- [find-prop-hw (lambda (name) (ast-find-child (lambda (i child) (string=? (->name (=real child)) name))
- (->Property* (car (->* (->ResourceType* (->HWRoot ast)))))))]
- [find-prop-sw (lambda (name comp) (ast-find-child (lambda (i child) (string=? (->name (=real child)) name))
- (->Property* comp)))]
- [load (find-prop-hw load-name)]
- [energy (find-prop-sw pn-energy (find-create-comp comp-nr))]
- [prev-p (and req-comp-nr (find-prop-sw (node-name "p" (list req-comp-nr)) (find-create-comp req-comp-nr)))]
- [this-p (find-prop-sw (node-name "p" (list comp-nr)) (find-create-comp comp-nr))]
- [clauses (filter (lambda (c) c) (list (:ReqClause mquat-spec load comp-max-eq load-f)
- (:ProvClause mquat-spec energy comp-max-eq energy-f)
- (:ProvClause mquat-spec this-p comp-max-eq prov-f)
- (and req-comp-nr (:ReqClause mquat-spec prev-p comp-max-eq prev-f))))]
- [new (:Mode mquat-spec (node-name "m" (list mode-nr impl-nr comp-nr)) clauses)])
- (rewrite-add (->Mode* impl) new) new))
- (define (comp-nr-of i) (ast-child-index (<=comp i)))
- (define (impl-nr-of i) (ast-child-index i))
- (define (make-new-modes)
- ; Add new mode to every impl
- (for-each (lambda (i) (let ([comp-nr (comp-nr-of i)])
- (add-mode comp-nr (impl-nr-of i) (+ 1 (length (->* (->Mode* i))))
- (if (eq? comp-nr 1) #f (- comp-nr 1)) (lambda _ 0.8) ;req-comp-nr load-f
- (lambda _ 20) (lambda _ 2) (lambda _ 7)))) ;energy-f prov-f prev-f
- (=every-impl ast)))
- (define (delete-odd-modes) (for-each (lambda (m) (rewrite-delete m)) (ret-odds (=every-mode ast))))
+ (define (add-comp ast comp-nr)
+ (debug "#create new comp" comp-nr)
+ (let* ([energy (ast-find-child (lambda (i child) (string=? (->name child) pn-energy)) (->RealProperty* (->SWRoot ast)))]
+ [new (:Comp mquat-spec (node-name "c" (list comp-nr)) (list) #f
+ (list (:RealProperty mquat-spec (node-name "p" (list comp-nr))
+ #f "runtime" 'increasing agg-sum)
+ (:PropertyRef mquat-spec energy)))])
+ (rewrite-add (->Comp* (->SWRoot ast)) new) new))
+ (define (find-create ast l prefix lon make-new)
+ (let ([name (node-name prefix lon)])
+ (or (ast-find-child (lambda (i child) (string=? (->name child) name)) l) (make-new))))
+ (define (find-create-comp ast comp-nr) (find-create ast (->Comp* (->SWRoot ast)) "c" (list comp-nr) (lambda _ (add-comp ast comp-nr))))
+ (define (add-impl ast comp-nr impl-nr reqcomps)
+ (debug "#create new impl" comp-nr impl-nr reqcomps)
+ (let ([new (:Impl mquat-spec (node-name "i" (list impl-nr comp-nr)) (list)
+ (map (lambda (nr) (find-create-comp ast nr)) reqcomps) #f #f)])
+ (rewrite-add (->Impl* (find-create-comp ast comp-nr)) new) new))
+ (define (find-create-impl ast comp-nr impl-nr reqcomps) (find-create ast (->Impl* (find-create-comp ast comp-nr)) "i"
+ (list impl-nr comp-nr)
+ (lambda _ (add-impl ast comp-nr impl-nr reqcomps))))
+ (define (add-mode ast comp-nr impl-nr mode-nr req-comp-nr load-f energy-f prov-f prev-f)
+ (debug "#create new mode" comp-nr impl-nr mode-nr req-comp-nr)
+ (let* ([impl (find-create-impl ast comp-nr impl-nr (if req-comp-nr (list req-comp-nr) (list)))]
+ [find-prop-hw (lambda (name) (ast-find-child (lambda (i child) (string=? (->name (=real child)) name))
+ (->Property* (car (->* (->ResourceType* (->HWRoot ast)))))))]
+ [find-prop-sw (lambda (name comp) (ast-find-child (lambda (i child) (string=? (->name (=real child)) name))
+ (->Property* comp)))]
+ [load (find-prop-hw load-name)]
+ [energy (find-prop-sw pn-energy (find-create-comp ast comp-nr))]
+ [prev-p (and req-comp-nr (find-prop-sw (node-name "p" (list req-comp-nr)) (find-create-comp ast req-comp-nr)))]
+ [this-p (find-prop-sw (node-name "p" (list comp-nr)) (find-create-comp ast comp-nr))]
+ [clauses (filter (lambda (c) c) (list (:ReqClause mquat-spec load comp-max-eq load-f)
+ (:ProvClause mquat-spec energy comp-max-eq energy-f)
+ (:ProvClause mquat-spec this-p comp-max-eq prov-f)
+ (and req-comp-nr (:ReqClause mquat-spec prev-p comp-max-eq prev-f))))]
+ [new (:Mode mquat-spec (node-name "m" (list mode-nr impl-nr comp-nr)) clauses)])
+ (rewrite-add (->Mode* impl) new) new))
+ (define (comp-nr-of i) (ast-child-index (<=comp i)))
+ (define (impl-nr-of i) (ast-child-index i))
+ (define (make-new-modes ast)
+ ; Add new mode to every impl
+ (for-each (lambda (i) (let ([comp-nr (comp-nr-of i)])
+ (add-mode ast comp-nr (impl-nr-of i) (+ 1 (length (->* (->Mode* i))))
+ (if (eq? comp-nr 1) #f (- comp-nr 1)) (lambda _ 0.8) ;req-comp-nr load-f
+ (lambda _ 20) (lambda _ 2) (lambda _ 7)))) ;energy-f prov-f prev-f
+ (=every-impl ast)))
+ (define (delete-odd-modes ast) (for-each (lambda (m) (rewrite-delete m)) (ret-odds (=every-mode ast))))
+ (define (run-sw-test id-s specs)
+ (let ([ast (cst id-s specs)])
(display id-s)
(rewrite-terminal 'config ast id-s)
(sit id-s "01-init" ast) (display+flush ".")
- (make-new-modes) (sit id-s "02-new-modes" ast) (display+flush ".")
- (make-new-modes) (sit id-s "03-new-modes" ast) (display+flush ".")
- (make-new-modes) (sit id-s "04-new-modes" ast) (display+flush ".")
- (delete-odd-modes) (sit id-s "05-del-modes" ast) (display+flush ".")
- (delete-odd-modes) (sit id-s "06-del-modes" ast) (display+flush ".")
- (delete-odd-modes) (sit id-s "07-del-modes" ast) (display+flush ".")))
+ (make-new-modes ast) (sit id-s "02-new-modes" ast) (display+flush ".")
+ (make-new-modes ast) (sit id-s "03-new-modes" ast) (display+flush ".")
+ (make-new-modes ast) (sit id-s "04-new-modes" ast) (display+flush ".")
+ (delete-odd-modes ast) (sit id-s "05-del-modes" ast) (display+flush ".")
+ (delete-odd-modes ast) (sit id-s "06-del-modes" ast) (display+flush ".")
+ (delete-odd-modes ast) (sit id-s "07-del-modes" ast) (display+flush ".")))
(define (run-update-test id-s specs)
(let* ([ast (cst id-s specs)]
@@ -268,6 +255,48 @@
(rw* rt "load" #f ast) (sit id-s "07-every-comp-rand" ast) (display+flush ".")
(rw* rt "load" #f ast) (sit id-s "08-every-comp-rand" ast) (display+flush ".")))
+ ; 01 init (4 sw component, N containers)
+ ; 02 change values of containers
+ ; 03 create new modes for each implementation
+ ; 04 change values of containers
+ ; 05 add new software component
+ ; 06 change values of containers
+ ; 07 remove some containers
+ ; 08 change values of containers
+ ; 09 (change nothing)
+ ; 10 add previously removed containers (from 07)
+ ; 11 remove some modes
+ (define (run-complex-test id-s specs)
+ (define (new-request-comp ast)
+ (let ([req-comp-nr (ast-child-index (->target (<=request ast)))]
+ [new-comp-nr (+ 1 (ast-num-children (->Comp* (->SWRoot ast))))])
+ ;params of add-mode: ast comp-nr impl-nr mode-nr req-comp-nr load-f energy-f prov-f prev-f
+ (add-mode ast new-comp-nr 1 1 ;comp-nr impl-nr mode-nr
+ req-comp-nr (lambda _ 0.8) ;req-comp-nr load-f
+ (lambda _ 20) (lambda _ 2) (lambda _ 7)) ;energy-f prov-f prev-f
+ ;adjust request to target new comp
+ (rewrite-terminal 'target (<=request ast) (find-create-comp ast new-comp-nr))))
+ (let* ([ast (cst id-s specs)]
+ [rt (ast-child 1 (->ResourceType* (->HWRoot ast)))]
+ [pe+parent (lambda (pe) (cons pe (<- pe)))]
+ [odd-pes (map pe+parent (ret-odds (=every-container ast)))])
+ (rewrite-terminal 'config ast id-s)
+ (display id-s)
+ (sit id-s "01-init" ast)
+ (rw* rt "load" #f ast) (sit id-s "02-every-comp-rand" ast) (display+flush ".")
+ (make-new-modes ast) (sit id-s "03-new-modes" ast) (display+flush ".")
+ (rw* rt "load" #f ast) (sit id-s "04-every-comp-rand" ast) (display+flush ".")
+ (new-request-comp ast) (sit id-s "05-new-comp" ast) (display+flush ".")
+ (rw* rt "load" #f ast) (sit id-s "06-every-comp-rand" ast) (display+flush ".")
+ (let ([removed (map (lambda (pp) (cons (rewrite-delete (car pp)) (cdr pp))) odd-pes)])
+ (sit id-s "07-del-odd-pes" ast) (display+flush ".")
+ (rw* rt "load" #f ast) (sit id-s "08-every-comp-rand" ast) (display+flush ".")
+ (sit id-s "09-no-change" ast) (display+flush ".")
+ ;rp = removed and parent
+ (for-each (lambda (rp) (rewrite-add (cdr rp) (car rp))) (reverse removed))
+ (rw* rt "load" #f ast) (sit id-s "10-add-odd-pes" ast) (display+flush "."))
+ (delete-odd-modes ast) (sit id-s "11-del-modes" ast) (display+flush ".")))
+
(define (print-usage) (error "measurement-cli-call" "No valid arguments found, use 'all', 'dirs', 'prefix', 'suffix' or a number of ids."))
(define (measurement-cli-call command-line)
diff --git a/ilp_measurement.py b/ilp_measurement.py
index 08e32c0..186aa21 100644
--- a/ilp_measurement.py
+++ b/ilp_measurement.py
@@ -9,10 +9,10 @@ try:
from fabric.contrib.console import confirm
except ImportError:
from fabric_workaround import task, red, lcd, green
-from utils import local_quiet, call_racket, call_larceny, assertTrue, assertTrueAssertion, secure_remove, merge_csv
-import properties
+import utils, properties
+from utils import local_quiet, call_racket, call_larceny, assertTrue, secure_remove
-assertTrue = assertTrueAssertion
+assertTrue = utils.assertTrueAssertion
gen_results = 'gen.csv'
gen_header = ['timestamp', 'impl', 'dir', 'step', 'ilp-gen'] # generation of ilp
@@ -27,8 +27,6 @@ all_att_header = ['dir', 'attname', 'normalex', 'normalcalled', 'flushedex', 'f
all_results = 'all.csv'
-change_kinds = 'profiling/kinds.json'
-
class timed(object):
def __enter__(self, msg = ' done in {0:.3f}s'):
self.start = timeit.default_timer()
@@ -291,6 +289,11 @@ def conflate_results(pathname = '*-*', skip_gen = False, skip_sol = False, impls
# sol-results
local_quiet('tail -qn +2 profiling/sol-header profiling/*/{0}> profiling/sol-glpk-results.csv'.format(sol_results), capture = False)
+@task
+def test():
+ for change in utils.change_kinds()['changes']:
+ local_quiet('tail -qn +2 profiling/*{0}*/{1} > profiling/splitted/att-{0}-results.csv'.format(change, att_results), capture = False)
+
@task
def clean(dryrun = False):
""" Remove all generated files """
@@ -302,7 +305,7 @@ def clean(dryrun = False):
dummy_values = {'timestamp': '2070-01-01T00:00:00.00', 'dir': 'dummy-001', 'step': '01-dummy', 'attname': 'dummyatt'}
@task(name = 'distinction-of-changes')
-def change_distinction():
+def distinction_of_changes():
def maybe_insert_dummy(f):
with open(f, 'a+') as fd:
header = next(fd)
@@ -311,8 +314,7 @@ def change_distinction():
keys = (key.strip() for key in header.split(','))
values = [dummy_values.get(key, '0') for key in keys]
fd.write(','.join(values))
- with open(change_kinds) as fd:
- d = json.load(fd)
+ d = utils.change_kinds()
unnormal = '-v -e ' + ' -e '.join((c for c in d['strategies'] if c != 'normal'))
def get_strategy_pattern(strategy):
return unnormal if strategy == 'normal' else '-e {0}'.format(strategy)
@@ -435,14 +437,16 @@ def setup(name = None, to_default = False):
def new_measurements():
name = execute(get_remote_measurements)
execute(incoporate_remote_measurements)
+ execute(conflate_results)
+ execute(distinction_of_changes)
@task(name = 'get-remote-measurements')
@hosts('rschoene@141.76.65.177')
def get_remote_measurements(rdir = '~/git/racr-mquat/'):
+ with cd(rdir):
+ run('fab measure.conflate-results:skip_sol=True')
rdir = os.path.join(rdir, 'profiling')
print rdir
- with open('profiling/kinds.json') as fd:
- d = json.load(fd)
with cd(rdir):
import time, datetime
name = 'm_{}.tar'.format(int(time.mktime(datetime.datetime.now().timetuple())))
@@ -465,9 +469,19 @@ def incoporate_remote_measurements(archive, dryrun = False):
sys.stdout.write('.')
d, name = os.path.split(f)
d = os.path.split(d)[1]
- merge_csv(os.path.join('profiling', d, name), f, dryrun = dryrun)
+ utils.merge_csv(os.path.join('profiling', d, name), f, dryrun = dryrun)
sys.stdout.write('\n')
+@task
+def factor(incremental, flushed, noncached):
+ def get_average_time(f):
+ with open(f) as fd:
+ r = csv.reader(fd)
+ values = [float(row[-1]) for row in r if not row[0].isalpha()]
+ return sum(values) / float(len(values))
+ avi, avf, avn = map(get_average_time, [incremental, flushed, noncached])
+ print avi, avf, avn
+
if __name__ == '__main__':
check()
help()
diff --git a/utils.py b/utils.py
index f40d521..2214428 100644
--- a/utils.py
+++ b/utils.py
@@ -1,6 +1,6 @@
-import sys, os, csv, shutil
+import sys, os, csv, shutil, json
try:
- from fabric.api import local, quiet, task
+ from fabric.api import local, quiet, task, env
except ImportError:
from fabric_workaround import local, quiet
from constants import racketBin, larcenyBin, racketExec, larcenyExec
@@ -15,6 +15,7 @@ def assertTrueExit(expr, msg):
sys.exit(0)
assertTrue = assertTrueExit
+change_kinds_fname = 'profiling/kinds.json'
def local_quiet(cmd, abort_on_stderr = False, capture = True):
""" Runs the command quietly (by default), asserts successful execution and returns stdout """
@@ -62,6 +63,10 @@ def secure_remove(spec, globbing = False, dryrun = False):
total += 1
return total
+def change_kinds(fname = change_kinds_fname):
+ with open(fname) as fd:
+ return json.load(fd)
+
@task
def merge_csv(f1, f2, primaryColumn1 = 0, primaryColumn2 = 0, dryrun = False):
""" Merges second csv file into first csv """
@@ -69,10 +74,16 @@ def merge_csv(f1, f2, primaryColumn1 = 0, primaryColumn2 = 0, dryrun = False):
print 'Merge {0} into {1}'.format(f2, f1)
dir1 = os.path.dirname(f1)
if not os.path.exists(dir1):
- print 'Creating directory "{}"'.format(os.path.dirname(f1))
+ if dryrun:
+ print 'Creating directory "{}"'.format(os.path.dirname(f1))
+ else:
+ sys.stdout.write('d({})'.format(os.path.basename(os.path.dirname(f1))))
os.mkdir(dir1)
if not os.path.exists(f1):
- print 'Copying "{0}" to "{1}"'.format(f2, f1)
+ if dryrun:
+ print 'Copying "{0}" to "{1}"'.format(f2, f1)
+ else:
+ sys.stdout.write('c')
shutil.copy(f2, f1)
return
with open(f1) as fd1, open(f2) as fd2:
@@ -103,3 +114,20 @@ def ccsv():
fd2.write('3,b3.1,c3.1\n')
fd2.write('4,b4,c4\n')
fd2.write('5,b5,c4\n')
+
+@task(name = 'kill-lines')
+def kill_lines(*files):
+ start, end, doit = int(env.get('start', 8)), int(env.get('end', 16)), bool(env.get('doit', False))
+ for f in files:
+ print f
+ with open(f, 'r') as fd:
+ lines = fd.readlines()
+ if not doit:
+ print ''.join(lines) + '\n'
+ lines = lines[0:start]+lines[end+1:]
+ if not doit:
+ print ''.join(lines)
+
+ if doit:
+ with open(f, 'w') as fd:
+ fd.write(''.join(lines))
--
GitLab