Support receiving data from Zip archives (#35)

Author: nichtich

README.md

@@ -171,7 +171,12 @@ List and get files of the stage directory of a terminology.
 
 #### POST /terminology/:id/receive
 
-Receive terminology data. The location of the data is going to be extracted from terminology metadata from BARTOC but this has not been implemented yet. For now pass query parameter `from` instead to locate an URL or the name of a file in the data directory. Format must be RDF/Turtle for file extension `.ttl` or `.nt`, otherwise RDF/XML.
+Receive terminology data. The location of the data is going to be extracted from terminology metadata from BARTOC but this has not been implemented yet. For now pass query parameter `from` instead to locate an URL or the name of a file in the data directory. File format can be: 
+
+- RDF/Turtle for file extension `.ttl` or `.nt`
+- RDF/XML for file extension `.rdf` or `.xml`
+- JSKOS as newline delimited JSON for file extension `.ndjson`
+- A ZIP archive containing RDF files for file extension `.zip`
 
 #### GET /terminology/:id/receive
 
@@ -241,7 +246,11 @@ List and get files of the stage directory of a collection.
 
 #### POST /collection/:id/receive
 
-Receive and process collection data. The location of the data is taken from collection metadata field `access` if existing. The location can be overridden with optional query parameter `from` with an URL or a file name from local data directory.
+Receive and process collection data. The location of the data is taken from collection metadata field `access` if existing. The location can be overridden with optional query parameter `from` with an URL or a file name from local data directory. File format can be: 
+
+- RDF/Turtle for file extension `.ttl` or `.nt`
+- RDF/XML for file extension `.rdf` or `.xml`
+- A ZIP archive containing RDF files for file extension `.zip`
 
 #### GET /collection/:id/receive
 

lib/mappings.py

@@ -60,10 +60,10 @@ def save_mappings_stage(self, id, name, data):
         stage.mkdir(exist_ok=True)
         # TODO: guess format from first line
         # process_jskos_mappings(self, source, target, log):
-        #fmt = ".ttl"
-        #file = f"{name}{fmt}"
-        #f = open(stage / file, "w")
-        #f.write(data)
+        # fmt = ".ttl"
+        # file = f"{name}{fmt}"
+        # f = open(stage / file, "w")
+        # f.write(data)
 
     def append(self, id, data):
         self.save_mappings_stage(id, "append")

lib/rdf.py

@@ -18,19 +18,6 @@ def jsonld2nt(doc, context):
     return jsonld.to_rdf(expanded, options={'format': 'application/n-quads'})
 
 
-# for pretty-printing
-# Namespace prefixes for pretty RDF/Turtle
-# prefixes = read_json(Path(__file__).parent.parent / 'prefixes.json')
-#
-# def to_rdf(doc, context):
-#    nquads = jsonld2nt(doc, context)
-#   g = Graph(bind_namespaces="none")
-#    for prefix, uri in prefixes.items():
-#        g.bind(prefix, Namespace(uri))
-#    g.parse(data=nquads, format='nquads')
-#   return g
-
-
 class TripleStore:
     def __init__(self, api):
         self.api = api
@@ -86,11 +73,9 @@ def sparql_to_rdf(binding):
             return Literal(binding['value'])
 
 
-rdfparser = lightrdf.Parser()
-
-
 def triple_iterator(source, log):
     """Recursively extract RDF triples from a file, directory and/or ZIP archive."""
+    rdfparser = lightrdf.Parser()
     for name, path, archive in walk(source):
         format = None
         if name.endswith(".ttl"):
@@ -115,7 +100,8 @@ def triple_iterator(source, log):
             base = f"file://{file}"
 
         try:
-            log.append(f"Extracting RDF from {file} as {format}")
+            log.append(f"Extracting RDF from {base} as {format}")
+            # TODO: pass errors as warnings to logger instead of STDERR
             for triple in rdfparser.parse(file, format=format):
                 yield triple
         except Exception as e:

lib/registry.py

@@ -169,6 +169,8 @@ def identify_source(self, id, source=None):
                 fmt = "xml"
             elif Path(source).suffix in [".ndjson"]:
                 fmt = "ndjson"
+            elif Path(source).suffix in [".zip", ".ZIP"]:
+                fmt = "zip"
 
         if not fmt:
             raise ClientError("Unknown data format")
@@ -200,6 +202,7 @@ def fetch_source(self, id, source, fmt):
         return (original, log)
 
     def preprocess_source(self, id, file, fmt, log):
+        """Returned file must be RDF or ZIP (with RDF)."""
         return file
 
     def receive_rdf(self, id, source, log):

tests/rdf.zip

@@ -6,6 +6,7 @@
 from pathlib import Path
 import pytest
 
+from rdflib import Graph
 from lib import TripleStore, read_json
 from app import app, init
 
@@ -35,6 +36,17 @@
     "partOf": [base]
 }
 
+# for pretty-printing
+# Namespace prefixes for pretty RDF/Turtle
+#def to_rdf(doc, context):
+#    prefixes = read_json(Path(__file__).parent.parent / 'prefixes.json')
+#    nquads = jsonld2nt(doc, context)
+#    g = Graph(bind_namespaces="none")
+#    for prefix, uri in prefixes.items():
+#        g.bind(prefix, Namespace(uri))
+#    g.parse(data=nquads, format='nquads')
+#    return g
+
 
 def count_graphs():
     query = "SELECT ?g (count(*) as ?t) { GRAPH ?g {?s ?p ?o} } GROUP BY ?g"
@@ -306,6 +318,13 @@ def test_api(client):
     client.post('/collection/3/load')
     assert sparql.query(query) == graph[:1]
 
+    assert client.post('/collection/3/receive?from=rdf.zip').status_code == 200
+    assert client.post('/collection/3/load').status_code == 200
+
+    query = f"SELECT ?e {{ GRAPH <{base}3> {{ ?e a ?t }} }} ORDER BY ?o"
+    res = [r["e"]["value"] for r in sparql.query(query)]
+    assert res == [f'https://example.org/e{i}' for i in [1, 2, 3, 4]]
+
     assert client.post(
         '/terminology/1644/receive?from=crm.ttl').status_code == 200
 
@@ -348,7 +367,7 @@ def test_mappings(client):
         f'Retrieving source {cwd}/tests/mappings.ndjson from data directory',
         'Converting JSKOS mappings to RDF mapping triples',
         'Processed 2 lines into 1 mappings',
-        f'Extracting RDF from {stage}/mappings/1/original.ttl as turtle',
+        f'Extracting RDF from file://{stage}/mappings/1/original.ttl as turtle',
         'Removed 0 triples, remaining 1 unique triples.',
         'done']