Feat/jr/harvest run

.github/workflows/build-test.yaml

@@ -0,0 +1,25 @@
+name: Build test
+
+permissions:
+  contents: read
+  packages: write
+
+on:
+  pull_request:
+    types: [ready_for_review, opened, reopened, synchronize]
+    branches:
+      - main
+
+jobs:
+  build-fdk-rdf-parse-event-publisher:
+    name: Build when pull request is created
+    if: ${{ github.actor != 'dependabot[bot]' &&  github.event.pull_request.draft == false }}
+    uses: Informasjonsforvaltning/workflows/.github/workflows/build-push.yaml@main
+    with:
+      app_name: fdk-rdf-parse-event-publisher
+      environment: test
+      java_version: '21'
+      coverage_file_path: ./target/site/jacoco/jacoco.xml
+    secrets:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/codeql.yaml

@@ -1,5 +1,9 @@
 name: Code scanning (CodeQL)
 
+permissions:
+  contents: read
+  security-events: write
+
 on:
   pull_request:
     types: [ ready_for_review, opened, reopened, synchronize ]

.github/workflows/deploy-prod&demo.yaml

@@ -1,12 +1,13 @@
 name: Deploy to prod&demo
+
 permissions:
   contents: read
   packages: write
 
-on:
-  push:
-    branches:
-      - main
+#on:
+#  push:
+#    branches:
+#      - main
 
 jobs:
   build-fdk-rdf-parse-event-publisher:

.github/workflows/deploy-staging.yaml

@@ -1,13 +1,14 @@
 name: Deploy to staging
+
 permissions:
   contents: read
   packages: write
 
-on:
-  pull_request:
-    types: [ ready_for_review, opened, reopened, synchronize ]
-    branches:
-      - main
+#on:
+#  pull_request:
+#    types: [ ready_for_review, opened, reopened, synchronize ]
+#    branches:
+#      - main
 
 jobs:
   build-fdk-rdf-parse-event-publisher:

kafka/schemas/no.fdk.concept.ConceptEvent.avsc

@@ -11,6 +11,8 @@
                 "symbols": ["CONCEPT_HARVESTED", "CONCEPT_REASONED", "CONCEPT_REMOVED"]
             }
         },
+        {"name": "harvestRunId", "type": ["null", "string"], "default": null},
+        {"name": "uri", "type": ["null", "string"], "default": null},
         {"name": "fdkId", "type": "string"},
         {"name": "graph", "type": "string"},
         {"name": "timestamp", "type": "long"}

kafka/schemas/no.fdk.event.EventEvent.avsc

@@ -11,6 +11,8 @@
                 "symbols": ["EVENT_HARVESTED", "EVENT_REASONED", "EVENT_REMOVED"]
             }
         },
+        {"name": "harvestRunId", "type": ["null", "string"], "default": null},
+        {"name": "uri", "type": ["null", "string"], "default": null},
         {"name": "fdkId", "type": "string"},
         {"name": "graph", "type": "string"},
         {"name": "timestamp", "type": "long"}

kafka/schemas/no.fdk.harvest.HarvestEvent.avsc

@@ -0,0 +1,103 @@
+{
+  "name": "HarvestEvent",
+  "namespace": "no.fdk.harvest",
+  "type": "record",
+  "fields": [
+    {
+      "name": "phase",
+      "type": {
+        "type": "enum",
+        "name": "HarvestPhase",
+        "symbols": [
+          "INITIATING",
+          "HARVESTING",
+          "REASONING",
+          "RDF_PARSING",
+          "RESOURCE_PROCESSING",
+          "SEARCH_PROCESSING",
+          "AI_SEARCH_PROCESSING",
+          "SPARQL_PROCESSING"
+        ]
+      }
+    },
+    {
+      "name": "runId",
+      "type": ["null", "string"],
+      "default": null, 
+      "doc": "Unique identifier (UUID) for the harvest run this event belongs to"
+    },
+    {
+      "name": "dataType",
+      "type": {
+        "type": "enum",
+        "name": "DataType",
+        "symbols": [
+          "concept",
+          "dataset",
+          "informationmodel",
+          "dataservice",
+          "publicService",
+          "event"
+        ]
+      }
+    },
+    {
+      "name": "dataSourceId",
+      "type": ["null", "string"],
+      "default": null
+    },
+    {
+      "name": "dataSourceUrl",
+      "type": ["null", "string"],
+      "default": null
+    },
+    {
+      "name": "acceptHeader",
+      "type": ["null", "string"],
+      "default": null
+    },
+    {
+      "name": "fdkId",
+      "type": ["null", "string"],
+      "default": null
+    },
+    {
+      "name": "resourceUri",
+      "type": ["null", "string"],
+      "default": null
+    },
+    {
+      "name": "startTime",
+      "type": ["null", "string"],
+      "default": null
+    },
+    {
+      "name": "endTime",
+      "type": ["null", "string"],
+      "default": null
+    },
+    {
+      "name": "errorMessage",
+      "type": ["null", "string"],
+      "default": null
+    },
+    {
+      "name": "changedResourcesCount",
+      "type": ["null", "int"],
+      "default": null,
+      "doc": "Number of changed resources (only for DATASOURCE_RESOURCE_EXTRACTION)"
+    },
+    {
+      "name": "unchangedResourcesCount",
+      "type": ["null", "int"],
+      "default": null,
+      "doc": "Number of unchanged resources (only for DATASOURCE_RESOURCE_EXTRACTION)"
+    },
+    {
+      "name": "removedResourcesCount",
+      "type": ["null", "int"],
+      "default": null,
+      "doc": "Number of removed resources (only for DATASOURCE_RESOURCE_EXTRACTION)"
+    }
+  ]
+}

kafka/schemas/no.fdk.rdf.parse.RdfParseEvent.avsc

@@ -11,6 +11,8 @@
                 "symbols": ["DATASET", "DATA_SERVICE", "CONCEPT", "INFORMATION_MODEL", "SERVICE", "EVENT"]
             }
         },
+        {"name": "harvestRunId", "type": ["null", "string"], "default": null},
+        {"name": "uri", "type": ["null", "string"], "default": null},
         {"name": "fdkId", "type": "string"},
         {"name": "data", "type": "string"},
         {"name": "timestamp", "type": "long"}

kafka/schemas/no.fdk.service.ServiceEvent.avsc

@@ -11,6 +11,8 @@
                 "symbols": ["SERVICE_HARVESTED", "SERVICE_REASONED", "SERVICE_REMOVED"]
             }
         },
+        {"name": "harvestRunId", "type": ["null", "string"], "default": null},
+        {"name": "uri", "type": ["null", "string"], "default": null},
         {"name": "fdkId", "type": "string"},
         {"name": "graph", "type": "string"},
         {"name": "timestamp", "type": "long"}