Add echo provider to vunnel (#815)

* Add echo provider to vunnel

Signed-off-by: Ori Zerah <[email protected]>

* change to echo rolling

Signed-off-by: Ori Zerah <[email protected]>

* change unit test folder to echo:rolling

Signed-off-by: Ori Zerah <[email protected]>

* chore: use match labels from main

Signed-off-by: Will Murphy <[email protected]>

* chore: type hint

Signed-off-by: Will Murphy <[email protected]>

* chore: alphabetize providers

Signed-off-by: Will Murphy <[email protected]>

---------

Signed-off-by: Ori Zerah <[email protected]>
Signed-off-by: Will Murphy <[email protected]>
Co-authored-by: Will Murphy <[email protected]>

Author: orizerah

README.md

@@ -13,6 +13,7 @@ Supported data sources:
 - Amazon (https://alas.aws.amazon.com/AL2/alas.rss & https://alas.aws.amazon.com/AL2022/alas.rss)
 - Azure (https://github.com/microsoft/AzureLinuxVulnerabilityData)
 - Debian (https://security-tracker.debian.org/tracker/data/json & https://salsa.debian.org/security-tracker-team/security-tracker/raw/master/data/DSA/list)
+- Echo (https://advisory.echohq.com/data.json)
 - GitHub Security Advisories (https://api.github.com/graphql)
 - NVD (https://services.nvd.nist.gov/rest/json/cves/2.0)
 - Oracle (https://linux.oracle.com/security/oval)
@@ -59,6 +60,7 @@ alpine
 amazon
 chainguard
 debian
+echo
 github
 mariner
 minimos

pyproject.toml

@@ -66,6 +66,8 @@ check_untyped_defs = true
 no_implicit_reexport = true
 disallow_untyped_defs = true
 ignore_missing_imports = true
+# Note: new files are expected to have type hints. Please do not add files to this
+# ignore list unless they are generated.
 exclude = '''(?x)(
     ^src/vunnel/providers/alpine/parser\.py$      # ported from enterprise, never had type hints
     | ^src/vunnel/providers/amazon/parser\.py$    # ported from enterprise, never had type hints

src/vunnel/cli/config.py

@@ -48,6 +48,7 @@ class Providers:
     bitnami: providers.bitnami.Config = field(default_factory=providers.bitnami.Config)
     chainguard: providers.chainguard.Config = field(default_factory=providers.chainguard.Config)
     debian: providers.debian.Config = field(default_factory=providers.debian.Config)
+    echo: providers.echo.Config = field(default_factory=providers.echo.Config)
     epss: providers.epss.Config = field(default_factory=providers.epss.Config)
     github: providers.github.Config = field(default_factory=providers.github.Config)
     kev: providers.kev.Config = field(default_factory=providers.kev.Config)

src/vunnel/providers/__init__.py

@@ -11,6 +11,7 @@
     bitnami,
     chainguard,
     debian,
+    echo,
     epss,
     github,
     kev,
@@ -35,6 +36,7 @@
     amazon.Provider.name(): amazon.Provider,
     bitnami.Provider.name(): bitnami.Provider,
     debian.Provider.name(): debian.Provider,
+    echo.Provider.name(): echo.Provider,
     github.Provider.name(): github.Provider,
     mariner.Provider.name(): mariner.Provider,
     nvd.Provider.name(): nvd.Provider,

src/vunnel/providers/echo/__init__.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
+
+from vunnel import provider, result, schema
+
+from .parser import Parser
+
+if TYPE_CHECKING:
+    import datetime
+
+
+@dataclass
+class Config:
+    runtime: provider.RuntimeConfig = field(
+        default_factory=lambda: provider.RuntimeConfig(
+            result_store=result.StoreStrategy.SQLITE,
+            existing_results=result.ResultStatePolicy.DELETE_BEFORE_WRITE,
+        ),
+    )
+    request_timeout: int = 125
+
+
+class Provider(provider.Provider):
+    __schema__ = schema.OSSchema()
+    __distribution_version__ = int(__schema__.major_version)
+
+    _url = "https://advisory.echohq.com/data.json"
+    _namespace = "echo"
+
+    def __init__(self, root: str, config: Config | None = None):
+        if not config:
+            config = Config()
+        super().__init__(root, runtime_cfg=config.runtime)
+        self.config = config
+
+        self.logger.debug(f"config: {config}")
+
+        self.parser = Parser(
+            workspace=self.workspace,
+            url=self._url,
+            namespace=self._namespace,
+            download_timeout=self.config.request_timeout,
+            logger=self.logger,
+        )
+
+        # this provider requires the previous state from former runs
+        provider.disallow_existing_input_policy(config.runtime)
+
+    @classmethod
+    def name(cls) -> str:
+        return "echo"
+
+    def update(self, last_updated: datetime.datetime | None) -> tuple[list[str], int]:
+        with self.results_writer() as writer:
+            # TODO: tech debt: on subsequent runs, we should only write new vulns (this currently re-writes all)
+            for release, vuln_dict in self.parser.get():
+                for vuln_id, record in vuln_dict.items():
+                    writer.write(
+                        identifier=os.path.join(f"{self._namespace.lower()}:{release.lower()}", vuln_id),
+                        schema=self.__schema__,
+                        payload=record,
+                    )
+
+        return [self._url], len(writer)

src/vunnel/providers/echo/parser.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+import copy
+import logging
+import os
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import orjson
+
+from vunnel.utils import http_wrapper as http
+from vunnel.utils import vulnerability
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
+    from vunnel import workspace
+
+
+class Parser:
+    _release_ = "rolling"
+    _advisories_dir = "echo-advisories"
+    _advisories_filename = "data.json"
+
+    def __init__(
+        self,
+        workspace: workspace.Workspace,
+        url: str,
+        namespace: str,
+        download_timeout: int = 125,
+        logger: logging.Logger | None = None,
+    ):
+        self.download_timeout = download_timeout
+        self.advisories_dir_path = Path(workspace.input_path) / self._advisories_dir
+        self.url = url
+        self.namespace = namespace
+
+        if not logger:
+            logger = logging.getLogger(self.__class__.__name__)
+        self.logger = logger
+
+    def _download(self) -> None:
+        """
+        Downloads echo advisories files
+        :return:
+        """
+        if not os.path.exists(self.advisories_dir_path):
+            os.makedirs(self.advisories_dir_path, exist_ok=True)
+
+        try:
+            self.logger.info(f"downloading {self.namespace} advisories {self.url}")
+            r = http.get(self.url, self.logger, stream=True, timeout=self.download_timeout)
+            file_path = self.advisories_dir_path / self._advisories_filename
+            with open(file_path, "wb") as fp:
+                for chunk in r.iter_content():
+                    fp.write(chunk)
+        except Exception:
+            self.logger.exception(f"Error downloading Echo advisories from {self.url}")
+            raise
+
+    def _normalize(self, release: str, data: dict[str, Any]) -> dict[str, dict[str, Any]]:
+        """
+        Normalize all the advisories entries into vulnerability payload records
+        :param release:
+        :param advisories_data_dict:
+        :return:
+        """
+
+        self.logger.debug("normalizing vulnerability data")
+
+        vuln_dict = {}
+        for package, package_cves in data.items():
+            for cve_id, cve_info in package_cves.items():
+                if cve_id not in vuln_dict:
+                    record = copy.deepcopy(vulnerability.vulnerability_element)
+                    record["Vulnerability"]["Name"] = cve_id
+                    record["Vulnerability"]["NamespaceName"] = self.namespace + ":" + str(release)
+                    reference_links = vulnerability.build_reference_links(cve_id)
+                    record["Vulnerability"]["Link"] = reference_links[0] if reference_links else ""
+                    record["Vulnerability"]["Severity"] = cve_info.get("severity", "Unknown")
+                    record["Vulnerability"]["FixedIn"] = []
+                    vuln_dict[cve_id] = record
+                cve_record = vuln_dict[cve_id]
+                cve_record["Vulnerability"]["FixedIn"].append(  # type: ignore[union-attr]
+                    {
+                        "Name": package,
+                        "Version": cve_info.get("fixed_version", ""),
+                        "VersionFormat": "dpkg",
+                        "NamespaceName": self.namespace + ":" + str(release),
+                    },
+                )
+        return vuln_dict
+
+    def get(self) -> Generator[tuple[str, dict[str, dict[str, Any]]], None, None]:
+        """
+        Download, load and normalize wolfi sec db and return a dict of release - list of vulnerability records
+        :return:
+        """
+        # download the data
+        self._download()
+        with open(f"{self.advisories_dir_path}/{self._advisories_filename}") as fh:
+            advisories_data_dict = orjson.loads(fh.read())
+
+        yield self._release_, self._normalize(self._release_, advisories_data_dict)

tests/quality/config.yaml

@@ -84,6 +84,17 @@ tests:
     validations:
       - *default-validations
 
+  - provider: echo
+    additional_providers:
+      - name: nvd
+        use_cache: true
+    images:
+      - ghcr.io/buildecho/scanner-test:latest@sha256:60557350ad6976dad3b88d891de8f090b20b3271c660272d30d44b5d07b23edc
+    expected_namespaces:
+      - echo:distro:echo:rolling
+    validations:
+      - *default-validations
+
   - provider: amazon
     validations:
       - <<: *default-validations

tests/quality/vulnerability-match-labels

@@ -245,6 +245,23 @@ def test_config(monkeypatch) -> None:
       result_store: sqlite
       skip_download: false
       skip_newer_archive_check: false
+  echo:
+    request_timeout: 125
+    runtime:
+      existing_input: keep
+      existing_results: delete-before-write
+      import_results_enabled: false
+      import_results_host: ''
+      import_results_path: providers/{provider_name}/listing.json
+      on_error:
+        action: fail
+        input: keep
+        results: keep
+        retry_count: 3
+        retry_delay: 5
+      result_store: sqlite
+      skip_download: false
+      skip_newer_archive_check: false
   epss:
     dataset: current
     request_timeout: 125