From 597da6411c2286565f42b7bd9bf063930d763a59 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 4 Feb 2026 02:05:38 +0200 Subject: [PATCH 1/6] Migrate RetireDotnet to Advisory V2 Signed-off-by: ziad hany --- vulnerabilities/importers/__init__.py | 2 + .../v2_importers/retiredotnet_importer.py | 140 ++++++++++++++++++ .../v2_importers/test_retiredotnet_imprter.py | 45 ++++++ .../tests/test_data/retiredotnet_v2/1.json | 37 +++++ .../retiredotnet_v2/expected_file.json | 22 +++ 5 files changed, 246 insertions(+) create mode 100644 vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py create mode 100644 vulnerabilities/tests/pipelines/v2_importers/test_retiredotnet_imprter.py create mode 100644 vulnerabilities/tests/test_data/retiredotnet_v2/1.json create mode 100644 vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index be084844b..fad9236b9 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -73,6 +73,7 @@ from vulnerabilities.pipelines.v2_importers import pypa_importer as pypa_importer_v2 from vulnerabilities.pipelines.v2_importers import pysec_importer as pysec_importer_v2 from vulnerabilities.pipelines.v2_importers import redhat_importer as redhat_importer_v2 +from vulnerabilities.pipelines.v2_importers import retiredotnet_importer as retiredotnet_importer_v2 from vulnerabilities.pipelines.v2_importers import ruby_importer as ruby_importer_v2 from vulnerabilities.pipelines.v2_importers import ubuntu_osv_importer as ubuntu_osv_importer_v2 from vulnerabilities.pipelines.v2_importers import vulnrichment_importer as vulnrichment_importer_v2 @@ -108,6 +109,7 @@ debian_importer_v2.DebianImporterPipeline, mattermost_importer_v2.MattermostImporterPipeline, apache_tomcat_v2.ApacheTomcatImporterPipeline, + retiredotnet_importer_v2.RetireDotnetImporterPipeline, ubuntu_osv_importer_v2.UbuntuOSVImporterPipeline, nvd_importer.NVDImporterPipeline, github_importer.GitHubAPIImporterPipeline, diff --git a/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py b/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py new file mode 100644 index 000000000..90a938a1c --- /dev/null +++ b/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py @@ -0,0 +1,140 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import json +import operator +import re +from itertools import groupby +from pathlib import Path + +from fetchcode.vcs import fetch_via_vcs +from packageurl import PackageURL +from univers.version_range import NugetVersionRange + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import AffectedPackageV2 +from vulnerabilities.importer import ReferenceV2 +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 +from vulnerabilities.utils import get_advisory_url + + +class RetireDotnetImporterPipeline(VulnerableCodeBaseImporterPipelineV2): + license_url = "https://github.com/RetireNet/Packages/blob/master/LICENSE" + spdx_license_expression = "MIT" + repo_url = "git+https://github.com/RetireNet/Packages/" + pipeline_id = "retiredotnet_importer_v2" + + @classmethod + def steps(cls): + return ( + cls.clone, + cls.collect_and_store_advisories, + cls.clean_downloads, + ) + + def clone(self): + self.log(f"Cloning `{self.repo_url}`") + self.vcs_response = fetch_via_vcs(self.repo_url) + + def advisories_count(self): + root = Path(self.vcs_response.dest_dir) / "Content" + return sum(1 for _ in root.rglob("*.json")) + + def collect_advisories(self): + base_path = Path(self.vcs_response.dest_dir) + vuln = base_path / "Content" + affected_packages = [] + + for file in vuln.glob("*.json"): + advisory_id = "RetireNet-" + file.stem + advisory_url = get_advisory_url( + file=file, + base_path=base_path, + url="https://github.com/RetireNet/Packages/blob/master/", + ) + with open(file) as f: + json_doc = json.load(f) + description = json_doc.get("description") or "" + aliases = self.vuln_id_from_desc(description) + + # group by package name `id` + packages = json_doc.get("packages") or [] + key_func = operator.itemgetter("id") + packages.sort(key=key_func) + grouped_packages = groupby(packages, key=key_func) + + for key, group in grouped_packages: + affected_versions = [] + fixed_versions = [] + + for pkg in list(group): + name = pkg.get("id") + if not name: + continue + + affected_version = pkg.get("affected") + if affected_version: + affected_versions.append(affected_version) + + fixed_version = pkg.get("fix") + if fixed_version: + fixed_versions.append(fixed_version) + + affected_version_range = None + if affected_versions: + affected_version_range = NugetVersionRange.from_versions(affected_versions) + + fixed_version_range = None + if fixed_versions: + fixed_version_range = NugetVersionRange.from_versions(affected_versions) + + if affected_packages: + affected_packages.append( + AffectedPackageV2( + package=PackageURL(type="nuget", name=name), + affected_version_range=affected_version_range, + fixed_version_range=fixed_version_range, + ) + ) + + link = json_doc.get("link") + if link: + vuln_reference = [ + ReferenceV2( + url=link, + ) + ] + + yield AdvisoryData( + advisory_id=advisory_id, + aliases=[aliases] if aliases else [], + summary=description, + affected_packages=affected_packages, + references_v2=vuln_reference, + url=advisory_url, + ) + + @staticmethod + def vuln_id_from_desc(desc): + cve_regex = re.compile(r"CVE-\d+-\d+") + res = cve_regex.search(desc) + if res: + return desc[res.start() : res.end()] + else: + return None + + def clean_downloads(self): + """Cleanup any temporary repository data.""" + if self.vcs_response: + self.log(f"Removing cloned repository") + self.vcs_response.delete() + + def on_failure(self): + """Ensure cleanup is always performed on failure.""" + self.clean_downloads() diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_retiredotnet_imprter.py b/vulnerabilities/tests/pipelines/v2_importers/test_retiredotnet_imprter.py new file mode 100644 index 000000000..8c352ee30 --- /dev/null +++ b/vulnerabilities/tests/pipelines/v2_importers/test_retiredotnet_imprter.py @@ -0,0 +1,45 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from pathlib import Path +from unittest.mock import Mock +from unittest.mock import patch + +import pytest + +from vulnerabilities.pipelines.v2_importers.retiredotnet_importer import ( + RetireDotnetImporterPipeline, +) +from vulnerabilities.tests import util_tests + +TEST_DATA = Path(__file__).parent.parent.parent / "test_data" / "retiredotnet_v2" + + +def test_vuln_id_from_desc(): + importer = RetireDotnetImporterPipeline() + gibberish = "xyzabcpqr123" * 50 + "\n" * 100 + res = importer.vuln_id_from_desc(gibberish) + assert res is None + + desc = "abcdef CVE-2002-1968 pqrstuvwxyz:_|-|" + res = importer.vuln_id_from_desc(desc) + assert res == "CVE-2002-1968" + + +@pytest.mark.django_db +def test_retiredotnet_advisories_per_file(): + pipeline = RetireDotnetImporterPipeline() + test_file = TEST_DATA / "1.json" + expected_file = TEST_DATA / "expected_file.json" + pipeline.vcs_response = Mock(dest_dir=TEST_DATA) + + with patch.object(Path, "glob", return_value=[test_file]): + result = [adv.to_dict() for adv in pipeline.collect_advisories()] + + util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/test_data/retiredotnet_v2/1.json b/vulnerabilities/tests/test_data/retiredotnet_v2/1.json new file mode 100644 index 000000000..7bdab96ea --- /dev/null +++ b/vulnerabilities/tests/test_data/retiredotnet_v2/1.json @@ -0,0 +1,37 @@ +{ + "link": "https://github.com/aspnet/Announcements/issues/359", + "description": "Microsoft Security Advisory CVE-2019-0982: ASP.NET Core Denial of Service Vulnerability", + "packages": [ + { + "id": "Microsoft.AspNetCore.SignalR.Protocols.MessagePack", + "affected": "1.0.0", + "fix": "1.0.11" + }, + { + "id": "Microsoft.AspNetCore.SignalR.Protocols.MessagePack", + "affected": "1.0.1", + "fix": "1.0.11" + }, + { + "id": "Microsoft.AspNetCore.SignalR.Protocols.MessagePack", + "affected": "1.0.2", + "fix": "1.0.11" + }, + { + "id": "Microsoft.AspNetCore.SignalR.Protocols.MessagePack", + "affected": "1.0.3", + "fix": "1.0.11" + }, + { + "id": "Microsoft.AspNetCore.SignalR.Protocols.MessagePack", + "affected": "1.0.4", + "fix": "1.0.11" + }, + + { + "id": "Microsoft.AspNetCore.SignalR.Protocols.MessagePack", + "affected": "1.1.0", + "fix": "1.1.5" + } + ] +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json b/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json new file mode 100644 index 000000000..99833609c --- /dev/null +++ b/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json @@ -0,0 +1,22 @@ +[ + { + "advisory_id": "RetireNet-1", + "aliases": [ + "CVE-2019-0982" + ], + "summary": "Microsoft Security Advisory CVE-2019-0982: ASP.NET Core Denial of Service Vulnerability", + "affected_packages": [], + "references_v2": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://github.com/aspnet/Announcements/issues/359" + } + ], + "patches": [], + "severities": [], + "date_published": null, + "weaknesses": [], + "url": "https://github.com/RetireNet/Packages/blob/master/1.json" + } +] \ No newline at end of file From dbd585ee2fe214f0cc8799eb32e76892c26a94cd Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 4 Feb 2026 02:24:50 +0200 Subject: [PATCH 2/6] Add missing affected_packages Signed-off-by: ziad hany --- .../v2_importers/retiredotnet_importer.py | 2 +- .../v2_importers/test_retiredotnet_imprter.py | 2 +- .../tests/test_data/retiredotnet_v2/1.json | 37 ---- .../tests/test_data/retiredotnet_v2/12.json | 176 ++++++++++++++++++ .../retiredotnet_v2/expected_file.json | 97 +++++++++- 5 files changed, 269 insertions(+), 45 deletions(-) delete mode 100644 vulnerabilities/tests/test_data/retiredotnet_v2/1.json create mode 100644 vulnerabilities/tests/test_data/retiredotnet_v2/12.json diff --git a/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py b/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py index 90a938a1c..a12461543 100644 --- a/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py +++ b/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py @@ -94,7 +94,7 @@ def collect_advisories(self): if fixed_versions: fixed_version_range = NugetVersionRange.from_versions(affected_versions) - if affected_packages: + if affected_version_range or fixed_version_range: affected_packages.append( AffectedPackageV2( package=PackageURL(type="nuget", name=name), diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_retiredotnet_imprter.py b/vulnerabilities/tests/pipelines/v2_importers/test_retiredotnet_imprter.py index 8c352ee30..3a66a14e3 100644 --- a/vulnerabilities/tests/pipelines/v2_importers/test_retiredotnet_imprter.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_retiredotnet_imprter.py @@ -35,7 +35,7 @@ def test_vuln_id_from_desc(): @pytest.mark.django_db def test_retiredotnet_advisories_per_file(): pipeline = RetireDotnetImporterPipeline() - test_file = TEST_DATA / "1.json" + test_file = TEST_DATA / "12.json" expected_file = TEST_DATA / "expected_file.json" pipeline.vcs_response = Mock(dest_dir=TEST_DATA) diff --git a/vulnerabilities/tests/test_data/retiredotnet_v2/1.json b/vulnerabilities/tests/test_data/retiredotnet_v2/1.json deleted file mode 100644 index 7bdab96ea..000000000 --- a/vulnerabilities/tests/test_data/retiredotnet_v2/1.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "link": "https://github.com/aspnet/Announcements/issues/359", - "description": "Microsoft Security Advisory CVE-2019-0982: ASP.NET Core Denial of Service Vulnerability", - "packages": [ - { - "id": "Microsoft.AspNetCore.SignalR.Protocols.MessagePack", - "affected": "1.0.0", - "fix": "1.0.11" - }, - { - "id": "Microsoft.AspNetCore.SignalR.Protocols.MessagePack", - "affected": "1.0.1", - "fix": "1.0.11" - }, - { - "id": "Microsoft.AspNetCore.SignalR.Protocols.MessagePack", - "affected": "1.0.2", - "fix": "1.0.11" - }, - { - "id": "Microsoft.AspNetCore.SignalR.Protocols.MessagePack", - "affected": "1.0.3", - "fix": "1.0.11" - }, - { - "id": "Microsoft.AspNetCore.SignalR.Protocols.MessagePack", - "affected": "1.0.4", - "fix": "1.0.11" - }, - - { - "id": "Microsoft.AspNetCore.SignalR.Protocols.MessagePack", - "affected": "1.1.0", - "fix": "1.1.5" - } - ] -} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/retiredotnet_v2/12.json b/vulnerabilities/tests/test_data/retiredotnet_v2/12.json new file mode 100644 index 000000000..5b21f8b26 --- /dev/null +++ b/vulnerabilities/tests/test_data/retiredotnet_v2/12.json @@ -0,0 +1,176 @@ +{ + "link": "https://github.com/aspnet/Announcements/issues/334", + "description": "Microsoft Security Advisory CVE-2019-0564: ASP.NET Core Denial of Service Vulnerability", + "packages": [ + { + "id": "Microsoft.AspNetCore.WebSockets", + "affected": "2.1.0", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.WebSockets", + "affected": "2.1.1", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.WebSockets", + "affected": "2.2.0", + "fix": "2.2.1" + }, + { + "id": "Microsoft.AspNetCore.Server.Kestrel.Core", + "affected": "2.1.0", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.Server.Kestrel.Core", + "affected": "2.1.1", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.Server.Kestrel.Core", + "affected": "2.1.2", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.Server.Kestrel.Core", + "affected": "2.1.3", + "fix": "2.1.7" + }, + { + "id": "System.Net.WebSockets.WebSocketProtocol", + "affected": "4.5.0", + "fix": "4.5.3" + }, + { + "id": "System.Net.WebSockets.WebSocketProtocol", + "affected": "4.5.1", + "fix": "4.5.3" + }, + { + "id": "System.Net.WebSockets.WebSocketProtocol", + "affected": "4.5.2", + "fix": "4.5.3" + }, + { + "id": "Microsoft.NETCore.App", + "affected": "2.1.0", + "fix": "2.1.7" + }, + { + "id": "Microsoft.NETCore.App", + "affected": "2.1.1", + "fix": "2.1.7" + }, + { + "id": "Microsoft.NETCore.App", + "affected": "2.1.2", + "fix": "2.1.7" + }, + { + "id": "Microsoft.NETCore.App", + "affected": "2.1.3", + "fix": "2.1.7" + }, + { + "id": "Microsoft.NETCore.App", + "affected": "2.1.4", + "fix": "2.1.7" + }, + { + "id": "Microsoft.NETCore.App", + "affected": "2.1.5", + "fix": "2.1.7" + }, + { + "id": "Microsoft.NETCore.App", + "affected": "2.1.6", + "fix": "2.1.7" + }, + { + "id": "Microsoft.NETCore.App", + "affected": "2.2.0", + "fix": "2.2.1" + }, + { + "id": "Microsoft.AspNetCore.App", + "affected": "2.1.0", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.App", + "affected": "2.1.1", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.App", + "affected": "2.1.2", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.App", + "affected": "2.1.3", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.App", + "affected": "2.1.4", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.App", + "affected": "2.1.5", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.App", + "affected": "2.1.6", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.App", + "affected": "2.2.0", + "fix": "2.2.1" + }, + { + "id": "Microsoft.AspNetCore.All", + "affected": "2.1.0", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.All", + "affected": "2.1.1", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.All", + "affected": "2.1.2", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.All", + "affected": "2.1.3", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.All", + "affected": "2.1.4", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.All", + "affected": "2.1.5", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.All", + "affected": "2.1.6", + "fix": "2.1.7" + }, + { + "id": "Microsoft.AspNetCore.All", + "affected": "2.2.0", + "fix": "2.2.1" + } + ] +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json b/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json index 99833609c..445bd4e5e 100644 --- a/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json +++ b/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json @@ -1,22 +1,107 @@ [ { - "advisory_id": "RetireNet-1", + "advisory_id": "RetireNet-12", "aliases": [ - "CVE-2019-0982" + "CVE-2019-0564" + ], + "summary": "Microsoft Security Advisory CVE-2019-0564: ASP.NET Core Denial of Service Vulnerability", + "affected_packages": [ + { + "package": { + "type": "nuget", + "namespace": "", + "name": "Microsoft.AspNetCore.All", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", + "fixed_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "nuget", + "namespace": "", + "name": "Microsoft.AspNetCore.App", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", + "fixed_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "nuget", + "namespace": "", + "name": "Microsoft.AspNetCore.Server.Kestrel.Core", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3", + "fixed_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3", + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "nuget", + "namespace": "", + "name": "Microsoft.AspNetCore.WebSockets", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.2.0", + "fixed_version_range": "vers:nuget/2.1.0|2.1.1|2.2.0", + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "nuget", + "namespace": "", + "name": "Microsoft.NETCore.App", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", + "fixed_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + }, + { + "package": { + "type": "nuget", + "namespace": "", + "name": "System.Net.WebSockets.WebSocketProtocol", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:nuget/4.5.0|4.5.1|4.5.2", + "fixed_version_range": "vers:nuget/4.5.0|4.5.1|4.5.2", + "introduced_by_commit_patches": [], + "fixed_by_commit_patches": [] + } ], - "summary": "Microsoft Security Advisory CVE-2019-0982: ASP.NET Core Denial of Service Vulnerability", - "affected_packages": [], "references_v2": [ { "reference_id": "", "reference_type": "", - "url": "https://github.com/aspnet/Announcements/issues/359" + "url": "https://github.com/aspnet/Announcements/issues/334" } ], "patches": [], "severities": [], "date_published": null, "weaknesses": [], - "url": "https://github.com/RetireNet/Packages/blob/master/1.json" + "url": "https://github.com/RetireNet/Packages/blob/master/12.json" } ] \ No newline at end of file From d0a5f8f789ad3d536be58845c98ea2d55d11af6e Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 4 Feb 2026 16:28:00 +0200 Subject: [PATCH 3/6] Update the pipeline to run once Update advisory_id Simplify grouped_packages structure Signed-off-by: ziad hany --- .../v2_importers/retiredotnet_importer.py | 44 +++++++++---------- .../retiredotnet_v2/expected_file.json | 34 +++++++------- 2 files changed, 38 insertions(+), 40 deletions(-) diff --git a/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py b/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py index a12461543..3acbbdf87 100644 --- a/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py +++ b/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py @@ -8,9 +8,8 @@ # import json -import operator import re -from itertools import groupby +from collections import defaultdict from pathlib import Path from fetchcode.vcs import fetch_via_vcs @@ -29,6 +28,7 @@ class RetireDotnetImporterPipeline(VulnerableCodeBaseImporterPipelineV2): spdx_license_expression = "MIT" repo_url = "git+https://github.com/RetireNet/Packages/" pipeline_id = "retiredotnet_importer_v2" + run_once = True @classmethod def steps(cls): @@ -52,7 +52,7 @@ def collect_advisories(self): affected_packages = [] for file in vuln.glob("*.json"): - advisory_id = "RetireNet-" + file.stem + advisory_id = "retiredotnet-" + file.stem advisory_url = get_advisory_url( file=file, base_path=base_path, @@ -64,40 +64,38 @@ def collect_advisories(self): aliases = self.vuln_id_from_desc(description) # group by package name `id` - packages = json_doc.get("packages") or [] - key_func = operator.itemgetter("id") - packages.sort(key=key_func) - grouped_packages = groupby(packages, key=key_func) - - for key, group in grouped_packages: - affected_versions = [] - fixed_versions = [] - - for pkg in list(group): - name = pkg.get("id") - if not name: - continue + # { pkg_id: {'affected_versions': [], 'fixed': []} } + grouped_packages = defaultdict( + lambda: {"affected_versions": [], "fixed_versions": []} + ) + for pkg in json_doc.get("packages") or []: + name = pkg.get("id") + if not name: + continue - affected_version = pkg.get("affected") - if affected_version: - affected_versions.append(affected_version) + affected_version = pkg.get("affected") + if affected_version: + grouped_packages[name]["affected_versions"].append(affected_version) - fixed_version = pkg.get("fix") - if fixed_version: - fixed_versions.append(fixed_version) + fixed_version = pkg.get("fix") + if fixed_version: + grouped_packages[name]["fixed_versions"].append(fixed_version) + for pkg in grouped_packages: affected_version_range = None + affected_versions = grouped_packages[pkg]["affected_versions"] if affected_versions: affected_version_range = NugetVersionRange.from_versions(affected_versions) fixed_version_range = None + fixed_versions = grouped_packages[pkg]["fixed_versions"] if fixed_versions: fixed_version_range = NugetVersionRange.from_versions(affected_versions) if affected_version_range or fixed_version_range: affected_packages.append( AffectedPackageV2( - package=PackageURL(type="nuget", name=name), + package=PackageURL(type="nuget", name=pkg), affected_version_range=affected_version_range, fixed_version_range=fixed_version_range, ) diff --git a/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json b/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json index 445bd4e5e..c4eb08605 100644 --- a/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json +++ b/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json @@ -1,6 +1,6 @@ [ { - "advisory_id": "RetireNet-12", + "advisory_id": "retiredotnet-12", "aliases": [ "CVE-2019-0564" ], @@ -10,13 +10,13 @@ "package": { "type": "nuget", "namespace": "", - "name": "Microsoft.AspNetCore.All", + "name": "Microsoft.AspNetCore.WebSockets", "version": "", "qualifiers": "", "subpath": "" }, - "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", - "fixed_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", + "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.2.0", + "fixed_version_range": "vers:nuget/2.1.0|2.1.1|2.2.0", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] }, @@ -24,13 +24,13 @@ "package": { "type": "nuget", "namespace": "", - "name": "Microsoft.AspNetCore.App", + "name": "Microsoft.AspNetCore.Server.Kestrel.Core", "version": "", "qualifiers": "", "subpath": "" }, - "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", - "fixed_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", + "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3", + "fixed_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] }, @@ -38,13 +38,13 @@ "package": { "type": "nuget", "namespace": "", - "name": "Microsoft.AspNetCore.Server.Kestrel.Core", + "name": "System.Net.WebSockets.WebSocketProtocol", "version": "", "qualifiers": "", "subpath": "" }, - "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3", - "fixed_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3", + "affected_version_range": "vers:nuget/4.5.0|4.5.1|4.5.2", + "fixed_version_range": "vers:nuget/4.5.0|4.5.1|4.5.2", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] }, @@ -52,13 +52,13 @@ "package": { "type": "nuget", "namespace": "", - "name": "Microsoft.AspNetCore.WebSockets", + "name": "Microsoft.NETCore.App", "version": "", "qualifiers": "", "subpath": "" }, - "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.2.0", - "fixed_version_range": "vers:nuget/2.1.0|2.1.1|2.2.0", + "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", + "fixed_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] }, @@ -66,7 +66,7 @@ "package": { "type": "nuget", "namespace": "", - "name": "Microsoft.NETCore.App", + "name": "Microsoft.AspNetCore.App", "version": "", "qualifiers": "", "subpath": "" @@ -80,13 +80,13 @@ "package": { "type": "nuget", "namespace": "", - "name": "System.Net.WebSockets.WebSocketProtocol", + "name": "Microsoft.AspNetCore.All", "version": "", "qualifiers": "", "subpath": "" }, - "affected_version_range": "vers:nuget/4.5.0|4.5.1|4.5.2", - "fixed_version_range": "vers:nuget/4.5.0|4.5.1|4.5.2", + "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", + "fixed_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] } From 74ffe5edf8aed220a83c9c2b1879d99c5d5d4bbb Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 4 Feb 2026 17:29:49 +0200 Subject: [PATCH 4/6] Fix a typo in fixed_version_range Signed-off-by: ziad hany --- .../pipelines/v2_importers/retiredotnet_importer.py | 2 +- .../test_data/retiredotnet_v2/expected_file.json | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py b/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py index 3acbbdf87..037067065 100644 --- a/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py +++ b/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py @@ -90,7 +90,7 @@ def collect_advisories(self): fixed_version_range = None fixed_versions = grouped_packages[pkg]["fixed_versions"] if fixed_versions: - fixed_version_range = NugetVersionRange.from_versions(affected_versions) + fixed_version_range = NugetVersionRange.from_versions(fixed_versions) if affected_version_range or fixed_version_range: affected_packages.append( diff --git a/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json b/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json index c4eb08605..a73905617 100644 --- a/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json +++ b/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json @@ -16,7 +16,7 @@ "subpath": "" }, "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.2.0", - "fixed_version_range": "vers:nuget/2.1.0|2.1.1|2.2.0", + "fixed_version_range": "vers:nuget/2.1.7|2.1.7|2.2.1", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] }, @@ -30,7 +30,7 @@ "subpath": "" }, "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3", - "fixed_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3", + "fixed_version_range": "vers:nuget/2.1.7|2.1.7|2.1.7|2.1.7", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] }, @@ -44,7 +44,7 @@ "subpath": "" }, "affected_version_range": "vers:nuget/4.5.0|4.5.1|4.5.2", - "fixed_version_range": "vers:nuget/4.5.0|4.5.1|4.5.2", + "fixed_version_range": "vers:nuget/4.5.3|4.5.3|4.5.3", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] }, @@ -58,7 +58,7 @@ "subpath": "" }, "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", - "fixed_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", + "fixed_version_range": "vers:nuget/2.1.7|2.1.7|2.1.7|2.1.7|2.1.7|2.1.7|2.1.7|2.2.1", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] }, @@ -72,7 +72,7 @@ "subpath": "" }, "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", - "fixed_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", + "fixed_version_range": "vers:nuget/2.1.7|2.1.7|2.1.7|2.1.7|2.1.7|2.1.7|2.1.7|2.2.1", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] }, @@ -86,7 +86,7 @@ "subpath": "" }, "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", - "fixed_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", + "fixed_version_range": "vers:nuget/2.1.7|2.1.7|2.1.7|2.1.7|2.1.7|2.1.7|2.1.7|2.2.1", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] } From 22fb93d28b54045c390281aac8c9f5fbe6301f6e Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 10 Feb 2026 15:46:36 +0200 Subject: [PATCH 5/6] Update retiredotnet to use the new AdvisoryDataV2 Drop duplicates in affected and fixed version range Signed-off-by: ziad hany --- .../v2_importers/retiredotnet_importer.py | 12 ++++++------ .../test_data/retiredotnet_v2/expected_file.json | 14 +++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py b/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py index 037067065..806920e71 100644 --- a/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py +++ b/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py @@ -16,7 +16,7 @@ from packageurl import PackageURL from univers.version_range import NugetVersionRange -from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import AdvisoryDataV2 from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import ReferenceV2 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 @@ -66,7 +66,7 @@ def collect_advisories(self): # group by package name `id` # { pkg_id: {'affected_versions': [], 'fixed': []} } grouped_packages = defaultdict( - lambda: {"affected_versions": [], "fixed_versions": []} + lambda: {"affected_versions": set(), "fixed_versions": set()} ) for pkg in json_doc.get("packages") or []: name = pkg.get("id") @@ -75,11 +75,11 @@ def collect_advisories(self): affected_version = pkg.get("affected") if affected_version: - grouped_packages[name]["affected_versions"].append(affected_version) + grouped_packages[name]["affected_versions"].add(affected_version) fixed_version = pkg.get("fix") if fixed_version: - grouped_packages[name]["fixed_versions"].append(fixed_version) + grouped_packages[name]["fixed_versions"].add(fixed_version) for pkg in grouped_packages: affected_version_range = None @@ -109,12 +109,12 @@ def collect_advisories(self): ) ] - yield AdvisoryData( + yield AdvisoryDataV2( advisory_id=advisory_id, aliases=[aliases] if aliases else [], summary=description, affected_packages=affected_packages, - references_v2=vuln_reference, + references=vuln_reference, url=advisory_url, ) diff --git a/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json b/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json index a73905617..7c48a66b5 100644 --- a/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json +++ b/vulnerabilities/tests/test_data/retiredotnet_v2/expected_file.json @@ -16,7 +16,7 @@ "subpath": "" }, "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.2.0", - "fixed_version_range": "vers:nuget/2.1.7|2.1.7|2.2.1", + "fixed_version_range": "vers:nuget/2.1.7|2.2.1", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] }, @@ -30,7 +30,7 @@ "subpath": "" }, "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3", - "fixed_version_range": "vers:nuget/2.1.7|2.1.7|2.1.7|2.1.7", + "fixed_version_range": "vers:nuget/2.1.7", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] }, @@ -44,7 +44,7 @@ "subpath": "" }, "affected_version_range": "vers:nuget/4.5.0|4.5.1|4.5.2", - "fixed_version_range": "vers:nuget/4.5.3|4.5.3|4.5.3", + "fixed_version_range": "vers:nuget/4.5.3", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] }, @@ -58,7 +58,7 @@ "subpath": "" }, "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", - "fixed_version_range": "vers:nuget/2.1.7|2.1.7|2.1.7|2.1.7|2.1.7|2.1.7|2.1.7|2.2.1", + "fixed_version_range": "vers:nuget/2.1.7|2.2.1", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] }, @@ -72,7 +72,7 @@ "subpath": "" }, "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", - "fixed_version_range": "vers:nuget/2.1.7|2.1.7|2.1.7|2.1.7|2.1.7|2.1.7|2.1.7|2.2.1", + "fixed_version_range": "vers:nuget/2.1.7|2.2.1", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] }, @@ -86,12 +86,12 @@ "subpath": "" }, "affected_version_range": "vers:nuget/2.1.0|2.1.1|2.1.2|2.1.3|2.1.4|2.1.5|2.1.6|2.2.0", - "fixed_version_range": "vers:nuget/2.1.7|2.1.7|2.1.7|2.1.7|2.1.7|2.1.7|2.1.7|2.2.1", + "fixed_version_range": "vers:nuget/2.1.7|2.2.1", "introduced_by_commit_patches": [], "fixed_by_commit_patches": [] } ], - "references_v2": [ + "references": [ { "reference_id": "", "reference_type": "", From 8afe6916f9110e505e22de7c503da241c7c68936 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 10 Feb 2026 15:58:42 +0200 Subject: [PATCH 6/6] Fix a comment typo Signed-off-by: ziad hany --- vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py b/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py index 806920e71..bb92ed884 100644 --- a/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py +++ b/vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py @@ -64,7 +64,7 @@ def collect_advisories(self): aliases = self.vuln_id_from_desc(description) # group by package name `id` - # { pkg_id: {'affected_versions': [], 'fixed': []} } + # { pkg_id: {'affected_versions': set(), 'fixed': set()} } grouped_packages = defaultdict( lambda: {"affected_versions": set(), "fixed_versions": set()} )