Migrate RetireDotnet to Advisory V2

vulnerabilities/importers/__init__.py

@@ -73,6 +73,7 @@
 from vulnerabilities.pipelines.v2_importers import pypa_importer as pypa_importer_v2
 from vulnerabilities.pipelines.v2_importers import pysec_importer as pysec_importer_v2
 from vulnerabilities.pipelines.v2_importers import redhat_importer as redhat_importer_v2
+from vulnerabilities.pipelines.v2_importers import retiredotnet_importer as retiredotnet_importer_v2
 from vulnerabilities.pipelines.v2_importers import ruby_importer as ruby_importer_v2
 from vulnerabilities.pipelines.v2_importers import vulnrichment_importer as vulnrichment_importer_v2
 from vulnerabilities.pipelines.v2_importers import xen_importer as xen_importer_v2
@@ -107,6 +108,7 @@
         debian_importer_v2.DebianImporterPipeline,
         mattermost_importer_v2.MattermostImporterPipeline,
         apache_tomcat_v2.ApacheTomcatImporterPipeline,
+        retiredotnet_importer_v2.RetireDotnetImporterPipeline,
         nvd_importer.NVDImporterPipeline,
         github_importer.GitHubAPIImporterPipeline,
         gitlab_importer.GitLabImporterPipeline,

vulnerabilities/pipelines/v2_importers/retiredotnet_importer.py

@@ -0,0 +1,138 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import json
+import re
+from collections import defaultdict
+from pathlib import Path
+
+from fetchcode.vcs import fetch_via_vcs
+from packageurl import PackageURL
+from univers.version_range import NugetVersionRange
+
+from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import AffectedPackageV2
+from vulnerabilities.importer import ReferenceV2
+from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+from vulnerabilities.utils import get_advisory_url
+
+
+class RetireDotnetImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
+    license_url = "https://github.com/RetireNet/Packages/blob/master/LICENSE"
+    spdx_license_expression = "MIT"
+    repo_url = "git+https://github.com/RetireNet/Packages/"
+    pipeline_id = "retiredotnet_importer_v2"
+    run_once = True
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.clone,
+            cls.collect_and_store_advisories,
+            cls.clean_downloads,
+        )
+
+    def clone(self):
+        self.log(f"Cloning `{self.repo_url}`")
+        self.vcs_response = fetch_via_vcs(self.repo_url)
+
+    def advisories_count(self):
+        root = Path(self.vcs_response.dest_dir) / "Content"
+        return sum(1 for _ in root.rglob("*.json"))
+
+    def collect_advisories(self):
+        base_path = Path(self.vcs_response.dest_dir)
+        vuln = base_path / "Content"
+        affected_packages = []
+
+        for file in vuln.glob("*.json"):
+            advisory_id = "retiredotnet-" + file.stem
+            advisory_url = get_advisory_url(
+                file=file,
+                base_path=base_path,
+                url="https://github.com/RetireNet/Packages/blob/master/",
+            )
+            with open(file) as f:
+                json_doc = json.load(f)
+                description = json_doc.get("description") or ""
+                aliases = self.vuln_id_from_desc(description)
+
+                # group by package name `id`
+                # { pkg_id: {'affected_versions': [], 'fixed': []} }
+                grouped_packages = defaultdict(
+                    lambda: {"affected_versions": [], "fixed_versions": []}
+                )
+                for pkg in json_doc.get("packages") or []:
+                    name = pkg.get("id")
+                    if not name:
+                        continue
+
+                    affected_version = pkg.get("affected")
+                    if affected_version:
+                        grouped_packages[name]["affected_versions"].append(affected_version)
+
+                    fixed_version = pkg.get("fix")
+                    if fixed_version:
+                        grouped_packages[name]["fixed_versions"].append(fixed_version)
+
+                for pkg in grouped_packages:
+                    affected_version_range = None
+                    affected_versions = grouped_packages[pkg]["affected_versions"]
+                    if affected_versions:
+                        affected_version_range = NugetVersionRange.from_versions(affected_versions)
+
+                    fixed_version_range = None
+                    fixed_versions = grouped_packages[pkg]["fixed_versions"]
+                    if fixed_versions:
+                        fixed_version_range = NugetVersionRange.from_versions(fixed_versions)
+
+                    if affected_version_range or fixed_version_range:
+                        affected_packages.append(
+                            AffectedPackageV2(
+                                package=PackageURL(type="nuget", name=pkg),
+                                affected_version_range=affected_version_range,
+                                fixed_version_range=fixed_version_range,
+                            )
+                        )
+
+                link = json_doc.get("link")
+                if link:
+                    vuln_reference = [
+                        ReferenceV2(
+                            url=link,
+                        )
+                    ]
+
+                yield AdvisoryData(
+                    advisory_id=advisory_id,
+                    aliases=[aliases] if aliases else [],
+                    summary=description,
+                    affected_packages=affected_packages,
+                    references_v2=vuln_reference,
+                    url=advisory_url,
+                )
+
+    @staticmethod
+    def vuln_id_from_desc(desc):
+        cve_regex = re.compile(r"CVE-\d+-\d+")
+        res = cve_regex.search(desc)
+        if res:
+            return desc[res.start() : res.end()]
+        else:
+            return None
+
+    def clean_downloads(self):
+        """Cleanup any temporary repository data."""
+        if self.vcs_response:
+            self.log(f"Removing cloned repository")
+            self.vcs_response.delete()
+
+    def on_failure(self):
+        """Ensure cleanup is always performed on failure."""
+        self.clean_downloads()

vulnerabilities/tests/pipelines/v2_importers/test_retiredotnet_imprter.py

@@ -0,0 +1,45 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from pathlib import Path
+from unittest.mock import Mock
+from unittest.mock import patch
+
+import pytest
+
+from vulnerabilities.pipelines.v2_importers.retiredotnet_importer import (
+    RetireDotnetImporterPipeline,
+)
+from vulnerabilities.tests import util_tests
+
+TEST_DATA = Path(__file__).parent.parent.parent / "test_data" / "retiredotnet_v2"
+
+
+def test_vuln_id_from_desc():
+    importer = RetireDotnetImporterPipeline()
+    gibberish = "xyzabcpqr123" * 50 + "\n" * 100
+    res = importer.vuln_id_from_desc(gibberish)
+    assert res is None
+
+    desc = "abcdef CVE-2002-1968 pqrstuvwxyz:_|-|"
+    res = importer.vuln_id_from_desc(desc)
+    assert res == "CVE-2002-1968"
+
+
+@pytest.mark.django_db
+def test_retiredotnet_advisories_per_file():
+    pipeline = RetireDotnetImporterPipeline()
+    test_file = TEST_DATA / "12.json"
+    expected_file = TEST_DATA / "expected_file.json"
+    pipeline.vcs_response = Mock(dest_dir=TEST_DATA)
+
+    with patch.object(Path, "glob", return_value=[test_file]):
+        result = [adv.to_dict() for adv in pipeline.collect_advisories()]
+
+    util_tests.check_results_against_json(result, expected_file)

vulnerabilities/tests/test_data/retiredotnet_v2/12.json

@@ -0,0 +1,176 @@
+{
+  "link": "https://github.com/aspnet/Announcements/issues/334",
+  "description": "Microsoft Security Advisory CVE-2019-0564: ASP.NET Core Denial of Service Vulnerability",
+  "packages": [
+    {
+      "id": "Microsoft.AspNetCore.WebSockets",
+      "affected": "2.1.0",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.AspNetCore.WebSockets",
+      "affected": "2.1.1",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.AspNetCore.WebSockets",
+      "affected": "2.2.0",
+      "fix": "2.2.1"
+    },
+    {
+      "id": "Microsoft.AspNetCore.Server.Kestrel.Core",
+      "affected": "2.1.0",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.AspNetCore.Server.Kestrel.Core",
+      "affected": "2.1.1",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.AspNetCore.Server.Kestrel.Core",
+      "affected": "2.1.2",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.AspNetCore.Server.Kestrel.Core",
+      "affected": "2.1.3",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "System.Net.WebSockets.WebSocketProtocol",
+      "affected": "4.5.0",
+      "fix": "4.5.3"
+    },
+    {
+      "id": "System.Net.WebSockets.WebSocketProtocol",
+      "affected": "4.5.1",
+      "fix": "4.5.3"
+    },
+    {
+      "id": "System.Net.WebSockets.WebSocketProtocol",
+      "affected": "4.5.2",
+      "fix": "4.5.3"
+    },
+    {
+      "id": "Microsoft.NETCore.App",
+      "affected": "2.1.0",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.NETCore.App",
+      "affected": "2.1.1",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.NETCore.App",
+      "affected": "2.1.2",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.NETCore.App",
+      "affected": "2.1.3",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.NETCore.App",
+      "affected": "2.1.4",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.NETCore.App",
+      "affected": "2.1.5",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.NETCore.App",
+      "affected": "2.1.6",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.NETCore.App",
+      "affected": "2.2.0",
+      "fix": "2.2.1"
+    },
+    {
+      "id": "Microsoft.AspNetCore.App",
+      "affected": "2.1.0",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.AspNetCore.App",
+      "affected": "2.1.1",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.AspNetCore.App",
+      "affected": "2.1.2",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.AspNetCore.App",
+      "affected": "2.1.3",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.AspNetCore.App",
+      "affected": "2.1.4",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.AspNetCore.App",
+      "affected": "2.1.5",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.AspNetCore.App",
+      "affected": "2.1.6",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.AspNetCore.App",
+      "affected": "2.2.0",
+      "fix": "2.2.1"
+    },
+    {
+      "id": "Microsoft.AspNetCore.All",
+      "affected": "2.1.0",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.AspNetCore.All",
+      "affected": "2.1.1",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.AspNetCore.All",
+      "affected": "2.1.2",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.AspNetCore.All",
+      "affected": "2.1.3",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.AspNetCore.All",
+      "affected": "2.1.4",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.AspNetCore.All",
+      "affected": "2.1.5",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.AspNetCore.All",
+      "affected": "2.1.6",
+      "fix": "2.1.7"
+    },
+    {
+      "id": "Microsoft.AspNetCore.All",
+      "affected": "2.2.0",
+      "fix": "2.2.1"
+    }
+  ]
+}