Merge pull request #214 from spcl/feature/nosql_storage

Add interface for NoSQL storage

Author: mcopik

.gitignore

@@ -8,6 +8,9 @@ python-venv
 cache*
 !cache.py
 
+minio-volume
+scylladb-volume
+
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.mypy.ini

@@ -27,6 +27,15 @@ ignore_missing_imports = True
 [mypy-google.cloud]
 ignore_missing_imports = True
 
+[mypy-google.cloud.logging]
+ignore_missing_imports = True
+
+[mypy-google.cloud.monitoring_v3]
+ignore_missing_imports = True
+
+[mypy-google.cloud.storage]
+ignore_missing_imports = True
+
 [mypy-google.api_core]
 ignore_missing_imports = True
 

benchmarks-data

@@ -1,5 +1,6 @@
 {
   "timeout": 10,
   "memory": 128,
-  "languages": ["python", "nodejs"]
+  "languages": ["python", "nodejs"],
+  "modules": []
 }

benchmarks/100.webapps/110.dynamic-html/config.json

@@ -5,10 +5,7 @@
     'large': 100000
 }
 
-def buckets_count():
-    return (0, 0)
-
-def generate_input(data_dir, size, benchmarks_bucket, input_paths, output_paths, upload_func):
+def generate_input(data_dir, size, benchmarks_bucket, input_paths, output_paths, upload_func, nosql_func):
     input_config = {'username': 'testname'} 
     input_config['random_len'] = size_generators[size]
     return input_config

benchmarks/100.webapps/110.dynamic-html/input.py

@@ -1,5 +1,6 @@
 {
   "timeout": 30,
   "memory": 128,
-  "languages": ["python", "nodejs"]
+  "languages": ["python", "nodejs"],
+  "modules": ["storage"]
 }

benchmarks/100.webapps/120.uploader/config.json

@@ -11,7 +11,7 @@
 def buckets_count():
     return (0, 1)
 
-def generate_input(data_dir, size, benchmarks_bucket, input_buckets, output_buckets, upload_func):
+def generate_input(data_dir, size, benchmarks_bucket, input_buckets, output_buckets, upload_func, nosql_func):
     input_config = {'object': {}, 'bucket': {}}
     input_config['object']['url'] = url_generators[size]
     input_config['bucket']['bucket'] = benchmarks_bucket

benchmarks/100.webapps/120.uploader/input.py

@@ -0,0 +1,11 @@
+{
+  "timeout": 30,
+  "memory": 128,
+  "languages": [
+    "python",
+    "nodejs"
+  ],
+  "modules": [
+    "nosql"
+  ]
+}

benchmarks/100.webapps/130.crud-api/config.json

@@ -0,0 +1,96 @@
+import uuid
+
+
+def allocate_nosql() -> dict:
+    return {"shopping_cart": {"primary_key": "cart_id", "secondary_key": "product_id"}}
+
+
+def generate_input(
+    data_dir, size, benchmarks_bucket, input_buckets, output_buckets, upload_func, nosql_upload
+):
+
+    input_config = {}
+
+    cart_id = str(uuid.uuid4().hex)
+    write_cart_id = str(uuid.uuid4().hex)
+
+    # Set initial data
+
+    nosql_upload(
+        "130.crud-api",
+        "shopping_cart",
+        {"name": "Gothic Game", "price": 42, "quantity": 2},
+        ("cart_id", cart_id),
+        ("product_id", "game-gothic"),
+    )
+    nosql_upload(
+        "130.crud-api",
+        "shopping_cart",
+        {"name": "Gothic 2", "price": 142, "quantity": 3},
+        ("cart_id", cart_id),
+        ("product_id", "game-gothic-2"),
+    )
+    nosql_upload(
+        "130.crud-api",
+        "shopping_cart",
+        {"name": "SeBS Benchmark", "price": 1000, "quantity": 1},
+        ("cart_id", cart_id),
+        ("product_id", "sebs-benchmark"),
+    )
+    nosql_upload(
+        "130.crud-api",
+        "shopping_cart",
+        {"name": "Mint Linux", "price": 0, "quantity": 5},
+        ("cart_id", cart_id),
+        ("product_id", "mint-linux"),
+    )
+
+    requests = []
+
+    if size == "test":
+        # retrieve a single entry
+        requests.append(
+            {
+                "route": "GET /cart/{id}",
+                "path": {"id": "game-gothic"},
+                "body": {
+                    "cart": cart_id,
+                },
+            }
+        )
+    elif size == "small":
+        requests.append(
+            {
+                "route": "GET /cart",
+                "body": {
+                    "cart": cart_id,
+                },
+            }
+        )
+    elif size == "large":
+        # add many new entries
+        for i in range(5):
+            requests.append(
+                {
+                    "route": "PUT /cart",
+                    "body": {
+                        "cart": write_cart_id,
+                        "product_id": f"new-id-{i}",
+                        "name": f"Test Item {i}",
+                        "price": 100 * i,
+                        "quantity": i,
+                    },
+                }
+            )
+            requests.append(
+                {
+                    "route": "GET /cart",
+                    "body": {
+                        "cart": write_cart_id,
+                    },
+                }
+            )
+
+    input_config["requests"] = requests
+
+    return input_config

benchmarks/100.webapps/130.crud-api/input.py

@@ -0,0 +1,67 @@
+from . import nosql
+
+nosql_client = nosql.nosql.get_instance()
+
+nosql_table_name = "shopping_cart"
+
+
+def add_product(cart_id: str, product_id: str, product_name: str, price: float, quantity: int):
+
+    nosql_client.insert(
+        nosql_table_name,
+        ("cart_id", cart_id),
+        ("product_id", product_id),
+        {"price": price, "quantity": quantity, "name": product_name},
+    )
+
+
+def get_products(cart_id: str, product_id: str):
+    return nosql_client.get(nosql_table_name, ("cart_id", cart_id), ("product_id", product_id))
+
+
+def query_products(cart_id: str):
+
+    res = nosql_client.query(
+        nosql_table_name,
+        ("cart_id", cart_id),
+        "product_id",
+    )
+
+    products = []
+    price_sum = 0
+    quantity_sum = 0
+    for product in res:
+
+        products.append(product["name"])
+        price_sum += product["price"]
+        quantity_sum += product["quantity"]
+
+    avg_price = price_sum / quantity_sum if quantity_sum > 0 else 0.0
+
+    return {"products": products, "total_cost": price_sum, "avg_price": avg_price}
+
+
+def handler(event):
+
+    results = []
+
+    for request in event["requests"]:
+
+        route = request["route"]
+        body = request["body"]
+
+        if route == "PUT /cart":
+            add_product(
+                body["cart"], body["product_id"], body["name"], body["price"], body["quantity"]
+            )
+            res = {}
+        elif route == "GET /cart/{id}":
+            res = get_products(body["cart"], request["path"]["id"])
+        elif route == "GET /cart":
+            res = query_products(body["cart"])
+        else:
+            raise RuntimeError(f"Unknown request route: {route}")
+
+        results.append(res)
+
+    return {"result": results}