Skip to content

Commit 53b84d1

Browse files
ziadhanyziadhany
authored
Fix test (#1206)
Fix disk storage structure Redefine the disk storage structure Add a test for write_vul_data Rename file extension from yaml to yml again Add Filter before prefetch_related Add paginated again Fix typo in export and rename files from yaml to yml Fix filename error , Remove / from filename Create a query for distinct ecosystems Try to improve export performance again Try to improve export performance by load all data in memory before start writing on disk Improve export vulnerablecode data performance Try to improve export performance Try to improve performance by adding pagination Fix filename for export files Add multiple parameterizes for create_sub_path test . Add new format for exporting vulnerablecode-data Add a test Fix export test with yaml format Change the export format from json to yaml Add test for export command Add test for write_vuln_data function Edit export.py , Fix missing attribute in vuln_data Export vulnerablecode-data Add new format for exporting vulnerablecode-data Add a test Fix export test with yaml format Change the export format from json to yaml Add test for export command Add test for write_vuln_data function Edit export.py , Fix missing attribute in vuln_data Export vulnerablecode-data Add new format for exporting vulnerablecode-data Add a test Fix export test with yaml format Change the export format from json to yaml Add test for export command Add test for write_vuln_data function Edit export.py , Fix missing attribute in vuln_data Export vulnerablecode-data Signed-off-by: ziadhany <[email protected]>
1 parent 1561efe commit 53b84d1

File tree

2 files changed

+273
-0
lines changed

2 files changed

+273
-0
lines changed

‎vulnerabilities/management/commands/export.py

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# VulnerableCode is a trademark of nexB Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6+
# See https://github.com/nexB/vulnerablecode for support or download.
7+
# See https://aboutcode.org for more information about nexB OSS projects.
8+
#
9+
import logging
10+
import os
11+
from hashlib import sha512
12+
from pathlib import Path
13+
14+
import saneyaml
15+
from django.core.management.base import BaseCommand
16+
from django.core.management.base import CommandError
17+
from packageurl import PackageURL
18+
19+
from vulnerabilities.models import Package
20+
21+
logger = logging.getLogger(__name__)
22+
23+
24+
class Command(BaseCommand):
25+
help = "export vulnerablecode data"
26+
27+
def add_arguments(self, parser):
28+
parser.add_argument("path")
29+
30+
def handle(self, *args, **options):
31+
if options["path"]:
32+
git_path = Path(options["path"])
33+
if not git_path.is_dir():
34+
raise CommandError("Please enter a valid path")
35+
36+
self.export_data(git_path)
37+
38+
self.stdout.write(self.style.SUCCESS("Successfully exported vulnerablecode data"))
39+
40+
def export_data(self, git_path):
41+
"""
42+
export vulnerablecode data
43+
by running `python manage.py export /path/vulnerablecode-data`
44+
"""
45+
self.stdout.write("Exporting vulnerablecode data")
46+
47+
ecosystems = [pkg.type for pkg in Package.objects.distinct("type")]
48+
49+
for ecosystem in ecosystems:
50+
package_files = {} # {"package path": "data" }
51+
vul_files = {} # {"vulnerability path": "data" }
52+
53+
for purl in (
54+
Package.objects.filter(type=ecosystem)
55+
.prefetch_related("vulnerabilities")
56+
.paginated()
57+
):
58+
purl_without_version = PackageURL(
59+
type=purl.type,
60+
namespace=purl.namespace,
61+
name=purl.name,
62+
)
63+
64+
# ./aboutcode-packages-ed5/maven/org.apache.log4j/log4j-core/versions/vulnerabilities.yml
65+
pkg_filepath = (
66+
f"./aboutcode-packages-{get_purl_hash(purl_without_version)}/{purl.type}/{purl.namespace}/{purl.name}"
67+
f"/versions/vulnerabilities.yml"
68+
)
69+
70+
package_data = {
71+
"purl": str(purl),
72+
"affected_by_vulnerabilities": [
73+
vuln.vulnerability_id for vuln in purl.affected_by
74+
],
75+
"fixing_vulnerabilities": [vuln.vulnerability_id for vuln in purl.fixing],
76+
}
77+
78+
if pkg_filepath in package_files:
79+
package_files[pkg_filepath]["versions"].append(package_data)
80+
else:
81+
package_files[pkg_filepath] = {
82+
"package": str(purl_without_version),
83+
"versions": [package_data],
84+
}
85+
86+
for vul in purl.vulnerabilities.all():
87+
vulnerability_id = vul.vulnerability_id
88+
# ./aboutcode-vulnerabilities-12/34/VCID-1223-3434-34343/VCID-1223-3434-34343.yml
89+
vul_filepath = (
90+
f"./aboutcode-vulnerabilities-{vulnerability_id[5:7]}/{vulnerability_id[10:12]}"
91+
f"/{vulnerability_id}/{vulnerability_id}.yml"
92+
)
93+
vul_files[vul_filepath] = {
94+
"vulnerability_id": vul.vulnerability_id,
95+
"aliases": [alias.alias for alias in vul.get_aliases],
96+
"summary": vul.summary,
97+
"severities": [severity for severity in vul.severities.values()],
98+
"references": [ref for ref in vul.references.values()],
99+
"weaknesses": [
100+
"CWE-" + str(weakness["cwe_id"]) for weakness in vul.weaknesses.values()
101+
],
102+
}
103+
104+
for items in [package_files, vul_files]:
105+
for filepath, data in items.items():
106+
create_file(filepath, git_path, data)
107+
108+
self.stdout.write(f"Successfully exported {ecosystem} data")
109+
110+
111+
def create_file(filepath, git_path, data):
112+
"""
113+
Check if the directories exist if it doesn't exist create a new one then Create the file
114+
./aboutcode-vulnerabilities-12/34/VCID-1223-3434-34343/VCID-1223-3434-34343.yml
115+
./aboutcode-packages-ed5/maven/org.apache.log4j/log4j-core/versions/vulnerabilities.yml
116+
./aboutcode-packages-ed5/maven/org.apache.log4j/log4j-core/versions/1.2.3/vulnerabilities.yml
117+
"""
118+
filepath = git_path.joinpath(filepath)
119+
dirname = os.path.dirname(filepath)
120+
os.makedirs(dirname, exist_ok=True)
121+
data = saneyaml.dump(data)
122+
with open(filepath, encoding="utf-8", mode="w") as f:
123+
f.write(data)
124+
125+
126+
def get_purl_hash(purl: PackageURL, length: int = 3) -> str:
127+
"""
128+
Return a short lower cased hash of a purl.
129+
https://github.com/nexB/purldb/pull/235/files#diff-a1fd023bd42d73f56019d540f38be711255403547add15108540d70f9948dd40R154
130+
"""
131+
purl_bytes = str(purl).encode("utf-8")
132+
short_hash = sha512(purl_bytes).hexdigest()[:length]
133+
return short_hash.lower()

‎vulnerabilities/tests/test_export.py

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
import os
2+
from io import StringIO
3+
from pathlib import Path
4+
from unittest import TestCase
5+
6+
import pytest
7+
import saneyaml
8+
from django.core.management import call_command
9+
from django.core.management.base import CommandError
10+
11+
from vulnerabilities.models import Alias
12+
from vulnerabilities.models import Package
13+
from vulnerabilities.models import PackageRelatedVulnerability
14+
from vulnerabilities.models import Vulnerability
15+
from vulnerabilities.models import VulnerabilityReference
16+
from vulnerabilities.models import VulnerabilityRelatedReference
17+
from vulnerabilities.models import VulnerabilitySeverity
18+
from vulnerabilities.models import Weakness
19+
20+
21+
@pytest.fixture
22+
def package(db):
23+
return Package.objects.create(
24+
type="generic", namespace="nginx", name="test", version="2", qualifiers={}, subpath=""
25+
)
26+
27+
28+
@pytest.fixture
29+
def vulnerability_reference():
30+
return VulnerabilityReference.objects.create(
31+
reference_id="fake",
32+
url=f"https://..",
33+
)
34+
35+
36+
@pytest.fixture
37+
def vulnerability_severity(vulnerability_reference):
38+
return VulnerabilitySeverity.objects.create(
39+
scoring_system="cvssv3_vector",
40+
value="CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H",
41+
reference_id=vulnerability_reference.id,
42+
)
43+
44+
45+
@pytest.fixture
46+
def vulnerability(db, vulnerability_reference, vulnerability_severity):
47+
vulnerability = Vulnerability.objects.create(
48+
vulnerability_id="VCID-pst6-b358-aaap",
49+
summary="test-vuln",
50+
)
51+
Alias.objects.create(alias=f"CVE-xxx-xxx-xx", vulnerability=vulnerability)
52+
53+
VulnerabilityRelatedReference.objects.create(
54+
reference=vulnerability_reference, vulnerability=vulnerability
55+
)
56+
57+
weakness = Weakness.objects.create(cwe_id=15)
58+
vulnerability.weaknesses.add(weakness)
59+
60+
return vulnerability
61+
62+
63+
@pytest.fixture
64+
def package_related_vulnerability(db, package, vulnerability):
65+
PackageRelatedVulnerability.objects.create(
66+
package=package,
67+
vulnerability=vulnerability,
68+
fix=False,
69+
)
70+
return package
71+
72+
73+
class TestExportCommand(TestCase):
74+
def test_missing_path(self):
75+
with pytest.raises(CommandError) as cm:
76+
call_command("export", stdout=StringIO())
77+
78+
err = str(cm)
79+
assert "Error: the following arguments are required: path" in err
80+
81+
def test_bad_path_fail_error(self):
82+
with pytest.raises(CommandError) as cm:
83+
call_command("export", "/bad path", stdout=StringIO())
84+
85+
err = str(cm)
86+
assert "Please enter a valid path" in err
87+
88+
89+
@pytest.mark.django_db
90+
def test_export_data(
91+
tmp_path, package_related_vulnerability, vulnerability_reference, vulnerability_severity
92+
):
93+
expected_vul = {
94+
"vulnerability_id": "VCID-pst6-b358-aaap",
95+
"aliases": ["CVE-xxx-xxx-xx"],
96+
"summary": "test-vuln",
97+
"severities": [
98+
{
99+
"id": vulnerability_severity.id,
100+
"reference_id": vulnerability_reference.id,
101+
"scoring_system": "cvssv3_vector",
102+
"value": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H",
103+
"scoring_elements": "",
104+
"published_at": "",
105+
}
106+
],
107+
"references": [
108+
{
109+
"id": vulnerability_reference.id,
110+
"url": "https://..",
111+
"reference_type": "",
112+
"reference_id": "fake",
113+
}
114+
],
115+
"weaknesses": ["CWE-15"],
116+
}
117+
expected_pkg = {
118+
"package": "pkg:generic/nginx/test",
119+
"versions": [
120+
{
121+
"purl": "pkg:generic/nginx/test@2",
122+
"affected_by_vulnerabilities": ["VCID-pst6-b358-aaap"],
123+
"fixing_vulnerabilities": [],
124+
},
125+
],
126+
}
127+
128+
call_command("export", tmp_path, stdout=StringIO())
129+
130+
vul_filepath = os.path.join(
131+
tmp_path,
132+
"./aboutcode-vulnerabilities-ps/b3/VCID-pst6-b358-aaap/VCID-pst6-b358-aaap.yml",
133+
)
134+
pkg_filepath = os.path.join(
135+
tmp_path,
136+
"./aboutcode-packages-2cf/generic/nginx/test/versions/vulnerabilities.yml",
137+
)
138+
139+
assert Path(vul_filepath).read_text() == saneyaml.dump(expected_vul)
140+
assert Path(pkg_filepath).read_text() == saneyaml.dump(expected_pkg)

0 commit comments

Comments
 (0)