Skip to content
Snippets Groups Projects
Commit 1c8ba298 authored by Antonio García-Domínguez's avatar Antonio García-Domínguez
Browse files

Removing reporting for now (ask Georg to reintroduce it later)

parent 4562d812
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
Showing
with 35 additions and 721 deletions
.gitignore
+ 5
− 0
View file @ 1c8ba298
......@@ -21,3 +21,8 @@
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
hs_err_pid*
.Rproj.user
.Rhistory
*.Rproj
Rplots.pdf
config/reporting.json deleted 100644 → 0
+ 0
− 22
View file @ 4562d812
{
"Dimensions": {
"X_Dimensions": {
"RunIndex": true
},
"Groups": {
"Tool": true
}
},
"Summarize_Functions": [
{
"Name": "Runtime",
"Phases": [
"Initialization",
"Load",
"Run"
],
"Metric": "Time"
}
],
"Extension": ["png", "pdf"]
}
models/OUT.bddmodel deleted 100644 → 0
+ 0
− 51
View file @ 4562d812
<?xml version="1.0" encoding="ISO-8859-1"?>
<BDD xmi:version="2.0" xmlns:xmi="http://www.omg.org/XMI" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="BDD" name="Test">
<ports xsi:type="InputPort" name="a" subtrees="//@tree"/>
<ports xsi:type="InputPort" name="b" subtrees="//@tree/@treeForOne/@treeForZero //@tree/@treeForZero"/>
<ports xsi:type="InputPort" name="c" subtrees="//@tree/@treeForOne/@treeForZero/@treeForOne //@tree/@treeForZero/@treeForOne //@tree/@treeForOne/@treeForZero/@treeForZero"/>
<ports xsi:type="InputPort" name="d" subtrees="//@tree/@treeForOne //@tree/@treeForZero/@treeForOne/@treeForZero"/>
<ports xsi:type="OutputPort" name="s" assignments="//@tree/@treeForZero/@treeForZero/@assignments.0 //@tree/@treeForZero/@treeForOne/@treeForZero/@treeForOne/@assignments.0 //@tree/@treeForZero/@treeForOne/@treeForZero/@treeForZero/@assignments.0 //@tree/@treeForOne/@treeForZero/@treeForOne/@treeForZero/@assignments.0 //@tree/@treeForZero/@treeForOne/@treeForOne/@assignments.0 //@tree/@treeForOne/@treeForZero/@treeForOne/@treeForOne/@assignments.0 //@tree/@treeForOne/@treeForZero/@treeForZero/@treeForZero/@assignments.0 //@tree/@treeForOne/@treeForOne/@assignments.0 //@tree/@treeForOne/@treeForZero/@treeForZero/@treeForOne/@assignments.0"/>
<tree xsi:type="Subtree" port="//@ports.0">
<treeForZero xsi:type="Subtree" port="//@ports.1">
<treeForZero xsi:type="Leaf">
<assignments value="false" port="//@ports.4"/>
</treeForZero>
<treeForOne xsi:type="Subtree" port="//@ports.2">
<treeForZero xsi:type="Subtree" port="//@ports.3">
<treeForZero xsi:type="Leaf">
<assignments value="true" port="//@ports.4"/>
</treeForZero>
<treeForOne xsi:type="Leaf">
<assignments value="false" port="//@ports.4"/>
</treeForOne>
</treeForZero>
<treeForOne xsi:type="Leaf">
<assignments value="false" port="//@ports.4"/>
</treeForOne>
</treeForOne>
</treeForZero>
<treeForOne xsi:type="Subtree" port="//@ports.3">
<treeForZero xsi:type="Subtree" port="//@ports.1">
<treeForZero xsi:type="Subtree" port="//@ports.2">
<treeForZero xsi:type="Leaf">
<assignments value="false" port="//@ports.4"/>
</treeForZero>
<treeForOne xsi:type="Leaf">
<assignments value="true" port="//@ports.4"/>
</treeForOne>
</treeForZero>
<treeForOne xsi:type="Subtree" port="//@ports.2">
<treeForZero xsi:type="Leaf">
<assignments value="true" port="//@ports.4"/>
</treeForZero>
<treeForOne xsi:type="Leaf">
<assignments value="false" port="//@ports.4"/>
</treeForOne>
</treeForOne>
</treeForZero>
<treeForOne xsi:type="Leaf">
<assignments value="false" port="//@ports.4"/>
</treeForOne>
</treeForOne>
</tree>
</BDD>
models/Test.bdd deleted 100644 → 0
+ 0
− 27
View file @ 4562d812
bdd Test (in a, in b, in c, in d, out s) {
subtree a {
F:subtree b {
F:leaf s = F
T:subtree c {
F:subtree d {
F:leaf s = T
T:leaf s = F
}
T:leaf s = F
}
}
T:subtree d {
F:subtree b {
F:subtree c {
F:leaf s = F
T:leaf s = T
}
T:subtree c {
F:leaf s = T
T:leaf s = F
}
}
T:leaf s = F
}
}
}
\ No newline at end of file
models/Test.tt deleted 100644 → 0
+ 0
− 14
View file @ 4562d812
truth table Test (in a, in b, in c, in d, out s)
{
{a=F, b=F, s=F}
{a=F, b=T, c=F, d=F, s=T}
{a=F, b=T, c=F, d=T, s=F}
{a=F, b=T, c=T, s=F}
{a=T, b=F, c=F, d=F, s=F}
{a=T, b=F, c=T, d=F, s=T}
{a=T, d=T, s=F}
{a=T, b=T, c=F, d=F, s=T}
{a=T, b=T, c=T, d=F, s=F}
}
output/output.csv
+ 30
− 30
View file @ 1c8ba298
Tool;Model;RunIndex;PhaseName;MetricName;MetricValue
EMFSolutionATL;Test.ttmodel;0;Initialization;Time;249333761
EMFSolutionATL;Test.ttmodel;0;Initialization;Memory;4465808
EMFSolutionATL;Test.ttmodel;0;Load;Time;98825086
EMFSolutionATL;Test.ttmodel;0;Load;Memory;3397752
EMFSolutionATL;Test.ttmodel;0;Run;Time;508493087
EMFSolutionATL;Test.ttmodel;0;Run;Memory;6999144
EMFSolutionATL;Test.ttmodel;1;Initialization;Time;260234832
EMFSolutionATL;Test.ttmodel;1;Initialization;Memory;4462160
EMFSolutionATL;Test.ttmodel;1;Load;Time;90493702
EMFSolutionATL;Test.ttmodel;1;Load;Memory;3397472
EMFSolutionATL;Test.ttmodel;1;Run;Time;530846181
EMFSolutionATL;Test.ttmodel;1;Run;Memory;6671944
EMFSolutionATL;Test.ttmodel;2;Initialization;Time;255388759
EMFSolutionATL;Test.ttmodel;2;Initialization;Memory;4461760
EMFSolutionATL;Test.ttmodel;2;Load;Time;99738660
EMFSolutionATL;Test.ttmodel;2;Load;Memory;3396248
EMFSolutionATL;Test.ttmodel;2;Run;Time;510622775
EMFSolutionATL;Test.ttmodel;2;Run;Memory;6996344
EMFSolutionATL;Test.ttmodel;3;Initialization;Time;250852033
EMFSolutionATL;Test.ttmodel;3;Initialization;Memory;4465808
EMFSolutionATL;Test.ttmodel;3;Load;Time;103538439
EMFSolutionATL;Test.ttmodel;3;Load;Memory;3397552
EMFSolutionATL;Test.ttmodel;3;Run;Time;507472534
EMFSolutionATL;Test.ttmodel;3;Run;Memory;6998032
EMFSolutionATL;Test.ttmodel;4;Initialization;Time;252983568
EMFSolutionATL;Test.ttmodel;4;Initialization;Memory;4465880
EMFSolutionATL;Test.ttmodel;4;Load;Time;89641844
EMFSolutionATL;Test.ttmodel;4;Load;Memory;3396760
EMFSolutionATL;Test.ttmodel;4;Run;Time;523037634
EMFSolutionATL;Test.ttmodel;4;Run;Memory;6998256
EMFSolutionATL;Test.ttmodel;0;Initialization;Time;317448904
EMFSolutionATL;Test.ttmodel;0;Initialization;Memory;4462120
EMFSolutionATL;Test.ttmodel;0;Load;Time;114286387
EMFSolutionATL;Test.ttmodel;0;Load;Memory;3397440
EMFSolutionATL;Test.ttmodel;0;Run;Time;631488104
EMFSolutionATL;Test.ttmodel;0;Run;Memory;6998336
EMFSolutionATL;Test.ttmodel;1;Initialization;Time;306326404
EMFSolutionATL;Test.ttmodel;1;Initialization;Memory;4465368
EMFSolutionATL;Test.ttmodel;1;Load;Time;107336214
EMFSolutionATL;Test.ttmodel;1;Load;Memory;3397104
EMFSolutionATL;Test.ttmodel;1;Run;Time;602822281
EMFSolutionATL;Test.ttmodel;1;Run;Memory;6671464
EMFSolutionATL;Test.ttmodel;2;Initialization;Time;382360695
EMFSolutionATL;Test.ttmodel;2;Initialization;Memory;4461880
EMFSolutionATL;Test.ttmodel;2;Load;Time;143144706
EMFSolutionATL;Test.ttmodel;2;Load;Memory;3397048
EMFSolutionATL;Test.ttmodel;2;Run;Time;615091477
EMFSolutionATL;Test.ttmodel;2;Run;Memory;6670960
EMFSolutionATL;Test.ttmodel;3;Initialization;Time;415906493
EMFSolutionATL;Test.ttmodel;3;Initialization;Memory;4461752
EMFSolutionATL;Test.ttmodel;3;Load;Time;191628098
EMFSolutionATL;Test.ttmodel;3;Load;Memory;3385624
EMFSolutionATL;Test.ttmodel;3;Run;Time;654773931
EMFSolutionATL;Test.ttmodel;3;Run;Memory;6671144
EMFSolutionATL;Test.ttmodel;4;Initialization;Time;489395092
EMFSolutionATL;Test.ttmodel;4;Initialization;Memory;4465568
EMFSolutionATL;Test.ttmodel;4;Load;Time;171123680
EMFSolutionATL;Test.ttmodel;4;Load;Memory;3397464
EMFSolutionATL;Test.ttmodel;4;Run;Time;776413319
EMFSolutionATL;Test.ttmodel;4;Run;Memory;6671304
reporting/README.md deleted 100644 → 0
+ 0
− 70
View file @ 4562d812
# TTC 2017 Smart Grids Benchmark reporting tools
## Dependencies
1. Install R: the `r-base`, `r-base-dev` packages are required. The code is tested with R 3.0 (you can check your version with the `R --version` command).
```bash
sudo apt-get install -y r-base r-base-dev
sudo apt-get install -y libssl-dev
```
1. Install the required R packages by issuing the following command:
```bash
cd reporting/
sudo R -f install.R
```
As a consequence, the `ggplot2`, `jsonlite`, `plyr` packages will be installed.
## Usage
1. Adjust the configuration parameters in the `config/reporting.json` file.
1. Use the `scripts/run.py -v` command.
1. Alternatively, execute the following command from the `reporting` directory:
`Rscript generate_diagrams.R`
### Configuration
* An example of the configuration file can be seen below:
```json
{
"Dimensions": {
"X_Dimensions": {
"Size": true,
"Iteration": true
},
"Groups": {
"Query": true,
"Tool": true
}
},
"Summarize_Functions": [
{
"Phases": [
"check",
"transformation"
],
"Metric": "time",
"Y_Label": "Time (ms)",
"Y_Axis_Scale": -6
},
{
"Phases": [
"result"
],
"Metric": "scalar",
"Y_Label": "Invalid elements",
"Y_Axis_Scale": 0
}
],
"Extension": ["png", "pdf"]
}
```
* An arbitrary number of functions can be declared.
* The dimension and group parameters can be changed independently from each other, which means for example every variable can be `true` at the same time. As a consequence, more diagrams will be generated.
reporting/check_results.R deleted 100644 → 0
+ 0
− 25
View file @ 4562d812
expected = read.csv2("../expected-results/results.csv")
actual = subset(read.csv2("../output/output.csv"), MetricName=="Elements")
tools = unique(actual$Tool)
for (tool in tools) {
tool_data = subset(actual, Tool==tool)
queries = unique(tool_data$View)
for (query in queries) {
query_data = subset(tool_data, View==query)
query_n = length(row.names(query_data))
for (i in 1:query_n) {
query.row = query_data[i,]
expected.row = subset(expected, ChangeSet==query.row$ChangeSet & View==query & Iteration==query.row$Iteration)
if (length(as.character(expected.row$MetricValue)) > 0) {
if (as.character(query.row$MetricValue) != as.character(expected.row$MetricValue)) {
print(paste(tool, "is wrong. Was ", query.row$MetricValue, "but expected", expected.row$MetricValue, "for change set", query.row$ChangeSet, "query", query, "iteration", query.row$Iteration))
}
} else {
print(paste("Warning:", tool, "produced the result", query.row$MetricValue, "but expected result is unavailable for change set", query.row$ChangeSet, "query", query, "iteration", query.row$Iteration))
}
}
}
}
reporting/constants.R deleted 100644 → 0
+ 0
− 30
View file @ 4562d812
resultsPath <- "../output/output.csv"
# y-axis scaling
scaleDivisor <- 10**6
# output folder
rootPath <- c("../diagrams/")
# plot settings
pointSize = 3
lineSize = 1
yLabel <- "Time (ms)"
yAxis <- "Log10"
labels <- list("1" = "1\n ",
"2" = "2\n ",
"4" = "4\n ",
"8" = "8\n ",
"16"= "16\n ",
"32"= "32\n ",
"64" = "64\n ",
"128" = "128\n ",
"256" = "256\n ",
"512" = "512\n ",
"1024" = "1024\n ",
"2048" = "2048\n ",
"4096" = "4096\n ",
"8192" = "8192\n ",
"16384" = "16384\n "
)
reporting/extract_results.R deleted 100644 → 0
+ 0
− 35
View file @ 4562d812
library("plyr", quietly=T, verbose=F, warn.conflicts=FALSE)
source("constants.R")
results <-read.csv2(resultsPath, header=TRUE)
changes <- unique(results$ChangeSet)
metricName <- "elements"
for (change in changes){
subData <- subset(results, ChangeSet == change)
queries <- unique(subData$Query)
for (query in queries){
fileName <- paste("../results/",change, "-", query, ".csv", sep="")
if (file.exists(fileName)){
file.remove(fileName)
}
firstRow <- list("Size")
subData2 <- subset(subData, Query == query)
iterations <- unique(subData2$Iteration)
for (iter in iterations){
firstRow <- paste(firstRow, iter, sep="\t")
}
write(firstRow, fileName, append=TRUE)
sizes <- unique(subData2$Size)
for (size in sizes){
row <- list(size)
for (iter in iterations){
metricValue <- subset(subData2, Size==size & Iteration==iter & MetricName==metricName & RunIndex==1)$MetricValue
row <- paste(row, metricValue, sep="\t")
}
write(row, fileName, append=TRUE)
}
}
}
reporting/functions.R deleted 100644 → 0
+ 0
− 125
View file @ 4562d812
source("theme.R")
savePlot <-function(results, settings, phases, fileName){
phases <- unlist(phases)
first <- TRUE
for(phase in phases){
if (first == TRUE){
merged <- subset(results, PhaseName == phase) # selection on phase
first <- FALSE
}
else{
merged <- rbind(merged, subset(results, PhaseName == phase)) # merge back
}
}
if (nrow(merged) == 0) {
return()
}
if (settings@xDimension == "ChangeSet"){
# summarise the iterations
data <- ddply(merged, c("ChangeSet", "View", "Tool", "RunIndex", "MetricName"),
summarize, MetricValue=sum(MetricValue))
data <- ddply(data, c("ChangeSet", "View", "Tool", "MetricName"),
summarize, MetricValue=mean(MetricValue))
}
else if(settings@xDimension == "Iteration"){
data <- ddply(merged, c("View", "Tool", "ChangeSet", "RunIndex", "MetricName", "Iteration"),
summarize, MetricValue=sum(MetricValue))
data <- ddply(data, c("View", "Tool", "ChangeSet", "MetricName", "Iteration"),
summarize, MetricValue=median(MetricValue))
}
else {
return()
}
artifacts <- unique(data[[settings@xDimension]])
minValue <- min(data$MetricValue)
maxValue <- max(data$MetricValue)
xLabels <- c(artifacts)
plot <- ggplot(data,aes_string(x = settings@xDimension, y = settings@yDimension)) +
geom_line(aes_string(group = settings@group, colour=settings@group), size=lineSize) +
geom_point(aes_string(shape = settings@group, colour=settings@group), size=pointSize) +
scale_shape_manual(values=1:nlevels(data[[settings@group]])) +
ylab(settings@yLabel) +
ggtitle(label = settings@title) +
bwTheme
if (settings@xAxis == "Continuous"){
plot <- plot + scale_x_continuous(breaks = c(artifacts))
}
else if (settings@xAxis == "Log2"){
plot <- plot + scale_x_log10(breaks = c(artifacts), labels = xLabels)
}
if (minValue == 0){
print("The minimum metricvalue equals with 0. The plot cannot be generated.")
return()
}
if (settings@yAxis == "Continuous"){
plot <- plot + scale_y_continuous(breaks = seq(minValue, maxValue, by=round(maxValue/5,0)),
labels = seq(minValue, maxValue, by=round(maxValue/5,0)))
}
else if (settings@yAxis == "Log10"){
plot <- plot + scale_y_log10(breaks = 10^seq(round(log10(minValue)), round(log10(maxValue)), by=1),
labels = 10^seq(round(log10(minValue)), round(log10(maxValue)), by=1))
}
ggsave(plot,filename = fileName, width=10, height=5, dpi=192)
print(fileName)
}
validPhase <- function(results, functions){
uniquePhases <- unique(results$PhaseName)
for(func in functions){
for(phase in func){
if(phase %in% uniquePhases == FALSE){
return(FALSE)
}
}
}
return(TRUE)
}
validMetric <- function(results, metric){
if (metric %in% levels(results$MetricName)){
return(TRUE)
}
return(FALSE)
}
createFolders <- function(rootPath, subFolders){
if (file.exists(rootPath) == FALSE){
dir.create(rootPath)
}
for (folder in subFolders){
path <- paste(rootPath, folder, sep='')
if (file.exists(path) == FALSE){
dir.create(path)
}
}
}
concatPhases <- function(phases){
phases <- unlist(phases)
first <- TRUE
for(p in phases){
if(first){
merged <- p
first <- FALSE
}
else{
merged <- paste(merged, p, sep='+')
}
}
return(merged)
}
getXLabels <- function(artifacts){
ticks <- c()
for(size in artifacts){
ticks <- c(ticks, labels[[as.character(size)]])
}
return(ticks)
}
reporting/install.R deleted 100644 → 0
+ 0
− 5
View file @ 4562d812
install.packages("ggplot2", repos="http://cran.rstudio.com/");
install.packages("jsonlite", repos="http://cran.rstudio.com/");
install.packages("devtools", repos="http://cran.rstudio.com/")
require(devtools)
install_version("plyr", version = "1.8.1", repos = "http://cran.rstudio.com/")
reporting/memory.r deleted 100644 → 0
+ 0
− 42
View file @ 4562d812
library("ggplot2")
library("plyr")
results <- read.csv2("../output/output.csv", row.names = NULL)
testset = ddply(results, c("Tool", "RunIndex", "ChangeSet", "View"), MetricValue=max(MetricValue))
testset = ddply(testset, c("Tool", "ChangeSet", "View"), summarize, Memory=mean(MetricValue)/(1024*1024))
pointSize = 3
lineSize = 1
bwTheme <- theme(text=element_text(family="Helvetica", size=16),
panel.background = element_rect(fill="#FFFFFF"),
legend.position="bottom",
legend.direction="horizontal",
panel.grid.major = element_line(size=0.3, colour="#333333"),
panel.grid.minor = element_line(size=0.15, colour="#CCCCCC"),
axis.text.x = element_text(colour="black"),
axis.text.y = element_text(colour="black"),
legend.position="right"
)
for (query in unique(testset$View)) {
data = subset(testset, View==query)
sizes <- unique(data$ChangeSet)
minValue <- min(data$Memory)
maxValue <- max(data$Memory)
plot <- ggplot(data, aes_string(x="ChangeSet",y="Memory"))
plot <- plot + geom_line(aes_string(group="Tool",colour="Tool"),size=lineSize)
plot <- plot + geom_point(aes_string(shape="Tool", colour="Tool"), size=pointSize)
plot <- plot + scale_shape_manual(values=1:4) + ylab("Memory (Mbyte)")
plot <- plot + xlab("ChangeSet")
plot <- plot + bwTheme
plot <- plot + scale_y_log10(breaks = 10^seq(round(log10(minValue)), round(log10(maxValue)), by=1),
labels = 10^seq(round(log10(minValue)), round(log10(maxValue)), by=1))
plot <- plot + ggtitle(paste(query, ", working set", sep=""))
ggsave(plot, filename=paste(query, "-memory.pdf", sep=""), width=7, height=4, dpi=192)
}
\ No newline at end of file
reporting/plot.R deleted 100644 → 0
+ 0
− 93
View file @ 4562d812
PlotSettings <- setClass(
"PlotSettings",
slots = c(
xDimension = "character",
yDimension = "character",
group = "character",
xLabel = "character",
yLabel = "character",
title = "character",
xAxis = "character",
yAxis = "character"
),
prototype=list(
xDimension = "",
yDimension = "",
group = "",
xLabel = "",
yLabel = "",
title = "",
xAxis = "",
yAxis = ""
)
)
setGeneric(name="setDimensions",
def=function(theObject, xDimension, yDimension){
standardGeneric("setDimensions")
}
)
setMethod(f="setDimensions",
signature="PlotSettings",
definition=function(theObject, xDimension, yDimension){
theObject@xDimension <- xDimension
theObject@yDimension <- yDimension
return(theObject)
})
setGeneric(name="setGroup",
def=function(theObject, group){
standardGeneric("setGroup")
}
)
setMethod(f="setGroup",
signature="PlotSettings",
definition=function(theObject, group){
theObject@group <- group
return(theObject)
})
setGeneric(name="setTitle",
def=function(theObject, title){
standardGeneric("setTitle")
}
)
setMethod(f="setTitle",
signature="PlotSettings",
definition=function(theObject, title){
theObject@title <- title
return(theObject)
})
setGeneric(name="setLabels",
def=function(theObject, xLabel, yLabel){
standardGeneric("setLabels")
}
)
setMethod(f="setLabels",
signature="PlotSettings",
definition=function(theObject, xLabel, yLabel){
theObject@xLabel <- xLabel
theObject@yLabel <- yLabel
return(theObject)
})
setGeneric(name="setAxis",
def=function(theObject, xAxis, yAxis){
standardGeneric("setAxis")
}
)
setMethod(f="setAxis",
signature="PlotSettings",
definition=function(theObject, xAxis, yAxis){
theObject@xAxis <- xAxis
theObject@yAxis <- yAxis
return(theObject)
})
reporting/theme.R deleted 100644 → 0
+ 0
− 9
View file @ 4562d812
bwTheme <- theme(text=element_text(family="Helvetica", size=16),
panel.background = element_rect(fill="#FFFFFF"),
legend.position="bottom",
legend.direction="horizontal",
panel.grid.major = element_line(size=0.3, colour="#333333"),
panel.grid.minor = element_line(size=0.15, colour="#CCCCCC"),
axis.text.x = element_text(colour="black"),
axis.text.y = element_text(colour="black")
)
\ No newline at end of file
reporting/visualize.R deleted 100644 → 0
+ 0
− 116
View file @ 4562d812
library("jsonlite", quietly=T, verbose=F, warn.conflicts=FALSE)
library("ggplot2",quietly=T, verbose=F, warn.conflicts=FALSE)
library("plyr", quietly=T, verbose=F, warn.conflicts=FALSE)
source("functions.R")
source("plot.R")
source("constants.R")
args <- commandArgs(trailingOnly = TRUE)
configPath <- args[1]
results <-read.csv2(resultsPath, header=TRUE, row.names = NULL)
results$Model = as.factor(results$Model)
config <- fromJSON(configPath)
if (validPhase(results, config$Summarize_Functions$Phases) == FALSE){
print("Non existing phasename provided!")
quit()
}
index <- 0
settings <- PlotSettings()
uniqueScenarios <- unique(results$Model)
for(row in 1:nrow(config$Summarize_Functions)){
phases <- config$Summarize_Functions[row,]$Phases
name <- config$Summarize_Functions[row,]$Name
index <- index + 1
for(scenario in uniqueScenarios){
metric <- "Time"
subData1 <- subset(results, Model==scenario & MetricName == metric)
subData1$MetricValue <- as.numeric(as.character(subData1$MetricValue)) * (10**-6)
if (config$Dimensions$Groups$View){
uniqueTools <- unique(subData1$Tool)
settings <- setGroup(settings, "View")
for(tool in uniqueTools){
subData2 <- subset(subData1, Tool==tool)
if (config$Dimensions$X_Dimensions$Model){
title <- paste(tool, ", ", scenario, ", Function: ", concatPhases(phases), sep='')
settings <- setTitle(settings, title)
settings <- setDimensions(settings, "Model", "MetricValue")
settings <- setLabels(settings, "Model", "Time (ms)")
settings <- setAxis(settings, "Factor", yAxis)
for (extension in config$Extension){
fileName <- paste(rootPath, scenario, "-", tool, "-GroupBy-View-", metric, "-", name, ".", extension, sep='')
savePlot(subData2, settings, phases, fileName)
}
write.csv(subData2, file = paste(rootPath, scenario, "-", tool, "-GroupBy-View-", metric, "-", name, ".csv", sep=''))
}
if (config$Dimensions$X_Dimensions$Iteration){
uniqueModels <-unique(subData2$Model)
for(changeSet in uniqueModels){
subData3 <- subset(subData2, Model==changeSet)
title <- paste(tool, ", Model: ", changeSet, ", Function: ", concatPhases(phases), sep='')
settings <- setTitle(settings, title)
settings <- setDimensions(settings, "Iteration", "MetricValue")
settings <- setLabels(settings, "Iterations", "Time (ms)")
settings <- setAxis(settings, "Continuous", yAxis)
for (extension in config$Extension){
fileName <- paste(rootPath, scenario, "-", tool, "-changeSet-", changeSet, "-GroupBy-View-", metric, "-", name, ".", extension, sep='')
savePlot(subData3, settings, phases, fileName)
}
write.csv(subData3, file = paste(rootPath, scenario, "-", tool, "-Model-", changeSet, "-GroupBy-View-", metric, "-", name, ".csv", sep=''))
}
}
}
}
}
if (config$Dimensions$Groups$Tool){
metric <- "Time"
subData1 <- subset(results, MetricName == metric)
subData1$MetricValue <- as.numeric(as.character(subData1$MetricValue)) * (10**-6)
uniqueViews <- unique(subData1$View)
settings <- setGroup(settings, "Tool")
for(view in uniqueViews){
subData2 <- subset(subData1, View==view)
if (config$Dimensions$X_Dimensions$Model){
title <- paste(view, ", Function: ", concatPhases(phases), sep='')
settings <- setTitle(settings, title)
settings <- setDimensions(settings, "Model", "MetricValue")
settings <- setLabels(settings, "Model", "Time (ms)")
settings <- setAxis(settings, "Discrete", yAxis)
for (extension in config$Extension){
fileName <- paste(rootPath, view, "-GroupBy-Tool-",metric, "-", name, ".", extension, sep='')
savePlot(subData2, settings, phases, fileName)
}
write.csv(ddply(subData2, c("Tool", "Model"), summarise, N=length(MetricValue), mean=mean(MetricValue), sd=sd(MetricValue)), file = paste(rootPath, view, "-GroupBy-Tool-",metric, "-", name, ".csv", sep=''))
}
if (config$Dimensions$X_Dimensions$Iteration){
uniqueSizes <-unique(subData2$Model)
settings <- setDimensions(settings, "Iteration", "MetricValue")
settings <- setLabels(settings, "Iterations", "Time (ms)")
settings <- setAxis(settings, "Continuous", yAxis)
for(size in uniqueSizes){
subData3 <- subset(subData2, Model==size)
title <- paste(view, ", Model: ", size, ", Function: ", concatPhases(phases), sep='')
for (extension in config$Extension){
fileName <- paste(rootPath, view, "-Model-", size, "-GroupBy-Tool-", metric, "-", name, ".", extension, sep='')
settings <- setTitle(settings, title)
savePlot(subData3, settings, phases, fileName)
}
write.csv(subData3, file = paste(rootPath, view, "-Model-", size, "-GroupBy-Tool-", metric, "-", name, ".csv", sep=''))
}
}
}
}
}
scripts/run.py
+ 0
− 27
View file @ 1c8ba298
......@@ -82,24 +82,6 @@ def set_working_directory(*path):
os.chdir(dir)
def visualize():
"""
Visualizes the benchmark results
"""
clean_dir("diagrams")
set_working_directory("reporting")
subprocess.call(["Rscript", "visualize.R", os.path.join(BASE_DIRECTORY, "config", "reporting.json")])
def extract_results():
"""
Extracts the benchmark results
"""
clean_dir("results")
set_working_directory("reporting")
subprocess.call(["Rscript", "check_results.R"])
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("-b", "--build",
......@@ -111,9 +93,6 @@ if __name__ == "__main__":
parser.add_argument("-s", "--skip-tests",
help="skip JUNIT tests",
action="store_true")
parser.add_argument("-v", "--visualize",
help="create visualizations",
action="store_true")
parser.add_argument("-e", "--extract",
help="extract results",
action="store_true")
......@@ -138,10 +117,6 @@ if __name__ == "__main__":
benchmark(config)
if args.test:
build(config, False)
if args.visualize:
visualize()
if args.extract:
extract_results()
# if there are no args, execute a full sequence
# with the test and the visualization/reporting
......@@ -149,5 +124,3 @@ if __name__ == "__main__":
if no_args:
build(config, False)
benchmark(config)
visualize()
extract_results()
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment