From cced021f35ffe057a8ae71131c94b14f53c8d5d7 Mon Sep 17 00:00:00 2001
From: Dmytro Pukhkaiev <dmytro.pukhkaiev@tu-dresden.de>
Date: Wed, 6 Feb 2019 16:55:59 +0100
Subject: [PATCH] Throw away indexes, when the score is too low
---
.../inf/st/mquat/eris/ErisMQuATBuilder.java | 2 +-
.../mquat/eris/coupling/ERISAccessPath.java | 15 ++
.../st/mquat/eris/coupling/ERISConnector.java | 188 ++++++++++++------
.../st/mquat/eris/coupling/ERISContainer.java | 4 +
4 files changed, 143 insertions(+), 66 deletions(-)
diff --git a/jastadd-mquat-eris/src/main/java/de/tudresden/inf/st/mquat/eris/ErisMQuATBuilder.java b/jastadd-mquat-eris/src/main/java/de/tudresden/inf/st/mquat/eris/ErisMQuATBuilder.java
index ce534aa..3328ef7 100644
--- a/jastadd-mquat-eris/src/main/java/de/tudresden/inf/st/mquat/eris/ErisMQuATBuilder.java
+++ b/jastadd-mquat-eris/src/main/java/de/tudresden/inf/st/mquat/eris/ErisMQuATBuilder.java
@@ -185,7 +185,7 @@ public class ErisMQuATBuilder {
}
Component directOrIndirectComponent = new Component();
- directOrIndirectComponent.setName(new Name("direct_or_indirect_" + createName(erisContainer, erisLivingPartition, attribute)));
+ directOrIndirectComponent.setName(new Name("directOrIndirect_" + createName(erisContainer, erisLivingPartition, attribute)));
Implementation directOrIndirectImplementation = new Implementation();
directOrIndirectImplementation.setName(new Name("directOrIndirectAccessPathSelector"));
diff --git a/jastadd-mquat-eris/src/main/java/de/tudresden/inf/st/mquat/eris/coupling/ERISAccessPath.java b/jastadd-mquat-eris/src/main/java/de/tudresden/inf/st/mquat/eris/coupling/ERISAccessPath.java
index 4c7020a..5153e38 100644
--- a/jastadd-mquat-eris/src/main/java/de/tudresden/inf/st/mquat/eris/coupling/ERISAccessPath.java
+++ b/jastadd-mquat-eris/src/main/java/de/tudresden/inf/st/mquat/eris/coupling/ERISAccessPath.java
@@ -17,6 +17,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.Scanner;
@@ -143,4 +144,18 @@ public class ERISAccessPath {
this.indexOrderPreserving = indexOrderPreserving;
}
+ public double getLookupTime(int att_count) {
+ // get the correct size
+ double lookuptime = 0;
+ int dist = Integer.MAX_VALUE;
+ for (ERISBenchmarkResult i : resultList) {
+ if (Math.abs(att_count - i.getDataSize()) < dist) {
+ dist = Math.abs(att_count - i.getDataSize());
+ lookuptime = i.getLookup();
+ }
+ }
+
+ return lookuptime;
+ }
+
}
diff --git a/jastadd-mquat-eris/src/main/java/de/tudresden/inf/st/mquat/eris/coupling/ERISConnector.java b/jastadd-mquat-eris/src/main/java/de/tudresden/inf/st/mquat/eris/coupling/ERISConnector.java
index a2eb96e..55afb75 100644
--- a/jastadd-mquat-eris/src/main/java/de/tudresden/inf/st/mquat/eris/coupling/ERISConnector.java
+++ b/jastadd-mquat-eris/src/main/java/de/tudresden/inf/st/mquat/eris/coupling/ERISConnector.java
@@ -217,7 +217,85 @@ public class ERISConnector {
*
*/
Map<String, Map<List<String>, List<String>>> partitionStorageConfigAttributes = new HashMap<>();
+ // go through direct access paths
+ boolean searchIndexes = false;
+ partitionStorageConfigAttributes = formOutputDataStructute(solution, partitionStorageConfigAttributes, searchIndexes);
+ // go through indexes and decide whether they are viable
+ searchIndexes = true;
+ partitionStorageConfigAttributes = formOutputDataStructute(solution, partitionStorageConfigAttributes, searchIndexes);
+ //Sort by Container / Node / Local --> APid - attr[]
+ /**
+ * POST /swctrl/lp/<container id>/<node id>/<local id>
+ *
+ * Body: {storageConfigurationGroups: [<group>, <group>]}
+ * <group>: {attributes: [<attr>, <attr>], newAPId: <id>, newAPisDynamic: bool, isIndex: bool }
+ * <attr>: {id: <id>}
+ *
+ */
+ JSONObject result = new JSONObject();
+
+
+
+ for (String path : partitionStorageConfigAttributes.keySet()) {
+ JSONArray storageConfigurationGroups = new JSONArray();
+ // for each living partition
+ for (List<String> accessPathList : partitionStorageConfigAttributes.get(path).keySet()) {
+ boolean configChanged = false;
+ for (String attributeId : partitionStorageConfigAttributes.get(path).get(accessPathList)) {
+ if (attributeId.equals(""))
+ continue;
+ JSONObject jsonGroup = new JSONObject();
+ jsonGroup.put("newAPId", Integer.valueOf(accessPathList.get(0)));
+ jsonGroup.put("newAPisDynamic", Boolean.valueOf(accessPathList.get(1)));
+ jsonGroup.put("isIndex", Boolean.valueOf(accessPathList.get(2)));
+ JSONArray jsonArrayAttributes = new JSONArray();
+ JSONObject attribute = new JSONObject();
+ attribute.put("id", Integer.valueOf(attributeId));
+ String key = path + "/" + attributeId;
+ String value = accessPathList.get(0) + "_" + accessPathList.get(1) + "_" + accessPathList.get(2);
+ if (this.currentAccessPaths.containsKey(key) && this.currentAccessPaths.get(key).contains(value)) {
+ } else {
+ configChanged = true;
+ }
+ jsonArrayAttributes.put(attribute);
+ jsonGroup.put("attributes", jsonArrayAttributes);
+ if (!jsonArrayAttributes.isEmpty() && configChanged) {
+ storageConfigurationGroups.put(jsonGroup);
+ }
+ }
+ }
+ result.put("storageConfigurationGroups", storageConfigurationGroups);
+
+ // send result
+ logger.info("Sending result: {}", result);
+ if (path.equals("Config/Linkage/"))
+ continue;
+ try {
+ StringEntity params = new StringEntity(result.toString());
+
+ URI uri = new URIBuilder()
+ .setHost(ERIS_URL)
+ .setScheme("http")
+ .setPath("swctrl/lp/" + path)
+ .setParameter("id", sessionID)
+ .build();
+ HttpPost post = new HttpPost(uri);
+ post.setEntity(params);
+
+ String serverResponse = writeResult(post);
+ logger.info("Server responded with: {}", serverResponse);
+
+ } catch (UnsupportedEncodingException | URISyntaxException e) {
+ logger.error("Getting living partition failed: {}", e.getMessage());
+ throw new RuntimeException(e);
+ }
+
+ }
+ }
+
+ private Map<String, Map<List<String>, List<String>>> formOutputDataStructute(Solution solution, Map<String,
+ Map<List<String>, List<String>>> partitionStorageConfigAttributes, boolean searchIndexes){
for (Assignment assignment : solution.getAssignmentList()) {
// logger.debug(assignment.getRequest().getTarget().getRef().name());
@@ -228,9 +306,9 @@ public class ERISConnector {
StringBuilder sb = new StringBuilder();
String containerNodePartitionPath;
String attributeId = "";
- int j = 0;
- if (attributeGlobal[0].equals("direct"))
- j = 1;
+ int j = 1;
+// if (attributeGlobal[0].equals("directOrIndirect" ))
+// j = 1;
for (int i = j; i < attributeGlobal.length; i++) {
if (i < 2 + j) {
// can be direct
@@ -251,6 +329,8 @@ public class ERISConnector {
boolean isDynamic = false;
boolean isIndex = false;
+
+
// parse access path
// can be dynamic 0
// accessPathName 0
@@ -273,9 +353,11 @@ public class ERISConnector {
isIndex = true;
}
}
+ }
+ if (isIndex != searchIndexes)
+ continue;
- }
if (!partitionStorageConfigAttributes.containsKey(containerNodePartitionPath)) {
Map<List<String>, List<String>> tempAccessPathAttributes = new HashMap<>();
@@ -286,6 +368,7 @@ public class ERISConnector {
tempAccessPathList.add(accessPathId);
tempAccessPathList.add(Boolean.toString(isDynamic));
tempAccessPathList.add(String.valueOf(isIndex));
+
tempAccessPathAttributes.put(tempAccessPathList, tempAttrList);
partitionStorageConfigAttributes.put(containerNodePartitionPath, tempAccessPathAttributes);
@@ -306,76 +389,49 @@ public class ERISConnector {
partitionStorageConfigAttributes.get(containerNodePartitionPath).get(tempAccessPathList).add(attributeId);
}
}
-
- }
- //Sort by Container / Node / Local --> APid - attr[]
- /**
- * POST /swctrl/lp/<container id>/<node id>/<local id>
- *
- * Body: {storageConfigurationGroups: [<group>, <group>]}
- * <group>: {attributes: [<attr>, <attr>], newAPId: <id>, newAPisDynamic: bool, isIndex: bool }
- * <attr>: {id: <id>}
- *
- */
- JSONObject result = new JSONObject();
-
-
-
- for (String path : partitionStorageConfigAttributes.keySet()) {
- JSONArray storageConfigurationGroups = new JSONArray();
- // for each living partition
- for (List<String> accessPathList : partitionStorageConfigAttributes.get(path).keySet()) {
- boolean configChanged = false;
- for (String attributeId : partitionStorageConfigAttributes.get(path).get(accessPathList)) {
- if (attributeId.equals(""))
- continue;
- JSONObject jsonGroup = new JSONObject();
- jsonGroup.put("newAPId", Integer.valueOf(accessPathList.get(0)));
- jsonGroup.put("newAPisDynamic", Boolean.valueOf(accessPathList.get(1)));
- jsonGroup.put("isIndex", Boolean.valueOf(accessPathList.get(2)));
- JSONArray jsonArrayAttributes = new JSONArray();
- JSONObject attribute = new JSONObject();
- attribute.put("id", Integer.valueOf(attributeId));
- String key = path + "/" + attributeId;
- String value = accessPathList.get(0) + "_" + accessPathList.get(1) + "_" + accessPathList.get(2);
- if (this.currentAccessPaths.containsKey(key) && this.currentAccessPaths.get(key).contains(value)) {
- } else {
- configChanged = true;
- }
- jsonArrayAttributes.put(attribute);
- jsonGroup.put("attributes", jsonArrayAttributes);
- if (!jsonArrayAttributes.isEmpty() && configChanged) {
- storageConfigurationGroups.put(jsonGroup);
+ if (searchIndexes == true) {
+ ERISContainer container = containers.stream().filter(c -> c.getId() == Integer.parseInt(attributeGlobal[1])).findFirst().get();
+ ERISAttribute attribute = container.getAttributeInstance(Integer.parseInt(attributeGlobal[3]), Integer.parseInt(attributeGlobal[4])); //container [1] lp [3] id [4]
+ double lookupTime = 0;
+ double score = assignment.computeObjective();
+ // must be a direct access path corresponding to the same attribute
+ for (List<String> key : partitionStorageConfigAttributes.get(containerNodePartitionPath).keySet()){
+ ArrayList<String> tempAccessPathList = new ArrayList<>();
+ tempAccessPathList.add(accessPathId);
+ tempAccessPathList.add(Boolean.toString(isDynamic));
+ tempAccessPathList.add(String.valueOf(isIndex));
+ if (!key.equals(tempAccessPathList)){
+ if (partitionStorageConfigAttributes.get(containerNodePartitionPath).get(key).contains(attributeId)){
+ ERISAccessPath accessPath = accessPaths.stream().filter(ap -> ap.getId() == Integer.parseInt(key.get(0))).findFirst().get();
+ lookupTime = accessPath.getLookupTime(attribute.getStatistics().getLookups());
+ }
}
}
- }
- result.put("storageConfigurationGroups", storageConfigurationGroups);
- // send result
- logger.info("Sending result: {}", result);
- if (path.equals("Config/Linkage/"))
- continue;
- try {
- StringEntity params = new StringEntity(result.toString());
+ int lookupCount = attribute.getStatistics().getLookups();
+// System.out.println("Index score: " + score);
+// System.out.println("Direct: " + (lookupTime * lookupCount));
+ score += lookupTime * lookupCount;
+ if (score < 0) {
- URI uri = new URIBuilder()
- .setHost(ERIS_URL)
- .setScheme("http")
- .setPath("swctrl/lp/" + path)
- .setParameter("id", sessionID)
- .build();
- HttpPost post = new HttpPost(uri);
- post.setEntity(params);
- String serverResponse = writeResult(post);
- logger.info("Server responded with: {}", serverResponse);
+ ArrayList<String> tempAccessPathList = new ArrayList<>();
+ tempAccessPathList.add(accessPathId);
+ tempAccessPathList.add(Boolean.toString(isDynamic));
+ tempAccessPathList.add(String.valueOf(isIndex));
- } catch (UnsupportedEncodingException | URISyntaxException e) {
- logger.error("Getting living partition failed: {}", e.getMessage());
- throw new RuntimeException(e);
+ partitionStorageConfigAttributes.get(containerNodePartitionPath).get(tempAccessPathList).remove(attributeId);
+ if (partitionStorageConfigAttributes.get(containerNodePartitionPath).get(tempAccessPathList).isEmpty()) {
+ partitionStorageConfigAttributes.get(containerNodePartitionPath).remove(tempAccessPathList);
+ if (partitionStorageConfigAttributes.get(containerNodePartitionPath).isEmpty()) {
+ partitionStorageConfigAttributes.remove(containerNodePartitionPath);
+ }
+ }
+ }
}
}
+ return partitionStorageConfigAttributes;
}
private String writeResult(HttpPost httpPost) {
@@ -401,4 +457,6 @@ public class ERISConnector {
public ERISModel getErisModel() {
return new ERISModel(containers, accessPaths);
}
+
+
}
diff --git a/jastadd-mquat-eris/src/main/java/de/tudresden/inf/st/mquat/eris/coupling/ERISContainer.java b/jastadd-mquat-eris/src/main/java/de/tudresden/inf/st/mquat/eris/coupling/ERISContainer.java
index 16bead4..c24bb67 100644
--- a/jastadd-mquat-eris/src/main/java/de/tudresden/inf/st/mquat/eris/coupling/ERISContainer.java
+++ b/jastadd-mquat-eris/src/main/java/de/tudresden/inf/st/mquat/eris/coupling/ERISContainer.java
@@ -72,4 +72,8 @@ public class ERISContainer {
public void setPartitionList(List<ERISLivingPartition> partitionList) {
this.partitionList = partitionList;
}
+
+ public ERISAttribute getAttributeInstance(int partitionId, int attributeId){
+ return attributeList.stream().filter(a -> a.getId() == attributeId).findFirst().get();
+ }
}
--
GitLab