feat: Documentation and plugin system for parsers

cve_bin_tool/cvedb.py

@@ -7,6 +7,7 @@
 from __future__ import annotations
 
 import asyncio
+import contextlib
 import datetime
 import json
 import logging
@@ -1193,3 +1194,11 @@ def fetch_from_mirror(self, mirror, pubkey, ignore_signature, log_signature_erro
         else:
             self.clear_cached_data()
             return -1
+
+    @contextlib.contextmanager
+    def with_cursor(self):
+        cursor = self.db_open_and_get_cursor()
+        try:
+            yield cursor
+        finally:
+            self.db_close()

cve_bin_tool/egg_updater.py

@@ -2,11 +2,12 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 import ast
+import importlib.util
 import os
 import sys
 from io import StringIO
 
-from setuptools import Distribution, find_packages
+from setuptools import Distribution
 
 try:
     from cve_bin_tool.version import VERSION
@@ -59,44 +60,21 @@ def update_egg() -> None:
     with StringIO() as f:
         cwd = os.getcwd()
         os.chdir(os.path.join(os.path.dirname(__file__), ".."))
+        setup_spec = importlib.util.spec_from_file_location(
+            "setup", os.path.join(os.path.dirname(__file__), "..", "setup.py")
+        )
+        setup_module = importlib.util.module_from_spec(setup_spec)
+        setup_spec.loader.exec_module(setup_module)
+        setup_kwargs = setup_module.setup_kwargs
         sys.stdout = f
         sys.stderr = f
-        dist = Distribution(
+        setup_kwargs.update(
             dict(
                 script_name="setup.py",
                 script_args=["egg_info"],
-                name="cve-bin-tool",
-                version=VERSION,
-                packages=find_packages(
-                    exclude=["locales", "presentation"],
-                ),
-                entry_points={
-                    "console_scripts": [
-                        "cve-bin-tool = cve_bin_tool.cli:main",
-                        "csv2cve = cve_bin_tool.csv2cve:main",
-                    ],
-                    "cve_bin_tool.checker": [
-                        "{} = cve_bin_tool.checkers.{}:{}".format(
-                            filename.replace(".py", ""),
-                            filename.replace(".py", ""),
-                            "".join(
-                                (filename.replace(".py", "") + " checker")
-                                .replace("_", " ")
-                                .title()
-                                .split()
-                            ),
-                        )
-                        for filename in os.listdir(
-                            os.path.join(
-                                os.path.abspath(os.path.dirname(__file__)),
-                                "checkers",
-                            )
-                        )
-                        if filename.endswith(".py") and "__init__" not in filename
-                    ],
-                },
             )
         )
+        dist = Distribution(setup_kwargs)
         dist.parse_command_line()
         dist.run_commands()
         sys.stdout = sys.__stdout__

cve_bin_tool/parsers/__init__.py

@@ -13,6 +13,7 @@
 from cve_bin_tool.util import ProductInfo, ScanInfo
 
 __all__ = [
+    "parse",
     "Parser",
     "java",
     "javascript",
@@ -25,6 +26,7 @@
     "php",
     "perl",
     "dart",
+    "env",
 ]
 
 

cve_bin_tool/parsers/dart.py

@@ -15,6 +15,10 @@ class DartParser(Parser):
     https://dart.dev/overview
     """
 
+    PARSER_MATCH_FILENAMES = [
+        "pubspec.lock",
+    ]
+
     def __init__(self, cve_db, logger):
         super().__init__(cve_db, logger)
         self.purl_pkg_type = "pub"

cve_bin_tool/parsers/env.py

@@ -0,0 +1,134 @@
+# Copyright (C) 2024 Intel Corporation
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import annotations
+
+import dataclasses
+import pathlib
+import re
+
+from packageurl import PackageURL
+
+from cve_bin_tool.parsers import Parser
+from cve_bin_tool.util import ProductInfo, ScanInfo
+
+
+@dataclasses.dataclass
+class EnvNamespaceConfig:
+    ad_hoc_cve_id: str
+    vendor: str
+    product: str
+    version: str
+    location: str = "/usr/local/bin/product"
+
+
+@dataclasses.dataclass
+class EnvConfig:
+    namespaces: dict[str, EnvNamespaceConfig]
+
+
+class EnvParser(Parser):
+    """
+    Parser for Python requirements files.
+    This parser is designed to parse Python requirements files (usually named
+    requirements.txt) and generate PURLs (Package URLs) for the listed packages.
+    """
+
+    PARSER_MATCH_FILENAMES = [
+        ".env",
+    ]
+
+    @staticmethod
+    def parse_file_contents(contents):
+        lines = list(
+            [
+                line
+                for line in contents.replace("\r\n", "\n").split("\n")
+                if line.strip() and line.startswith("CVE_BIN_TOOL_")
+            ]
+        )
+        namespaces = {}
+        for i, line in enumerate(lines):
+            key, value = line.split("=", maxsplit=1)
+            namespace, key = key[len("CVE_BIN_TOOL_") :].split("_", maxsplit=1)
+            if value.startswith('"'):
+                value = value[1:]
+            if value.endswith('"'):
+                value = value[:-1]
+            namespaces.setdefault(namespace, {})
+            namespaces[namespace][key.lower()] = value
+        for namespace, config in namespaces.items():
+            namespaces[namespace] = EnvNamespaceConfig(**config)
+        return EnvConfig(namespaces=namespaces)
+
+    def run_checker(self, filename):
+        """
+        Parse the .env file and yield ScanInfo objects for the listed packages.
+        Args:
+            filename (str): The path to the .env file.
+        Yields:
+            str: ScanInfo objects for the packages listed in the file.
+        """
+        self.filename = filename
+        contents = pathlib.Path(self.filename).read_text()
+
+        env_config = self.parse_file_contents(contents)
+
+        data_source = "environment"
+        affected_data = [
+            {
+                "cve_id": cve.ad_hoc_cve_id,
+                "vendor": cve.vendor,
+                "product": cve.product,
+                # TODO Version MUST be unique to this bug!
+                "version": cve.version,
+                "versionStartIncluding": "",
+                # "versionStartIncluding": cve.version,
+                "versionStartExcluding": "",
+                "versionEndIncluding": "",
+                # "versionEndIncluding": cve.version,
+                "versionEndExcluding": "",
+            }
+            for _namespace, cve in env_config.namespaces.items()
+        ]
+        severity_data = [
+            {
+                "ID": cve.ad_hoc_cve_id,
+                # TODO severity
+                "severity": "LOW",
+                # TODO description
+                "description": "TODO",
+                # TODO score
+                "score": 0,
+                # TODO CVSS_version
+                "CVSS_version": 3,
+                # TODO CVSS_vector
+                "CVSS_vector": "",
+                "last_modified": "",
+            }
+            for _namespace, cve in env_config.namespaces.items()
+        ]
+
+        with self.cve_db.with_cursor() as cursor:
+            self.cve_db.populate_cve_metrics(severity_data, cursor)
+            self.cve_db.populate_severity(severity_data, cursor, data_source)
+            self.cve_db.populate_affected(affected_data, cursor, data_source)
+
+        for _namespace, cve in env_config.namespaces.items():
+            yield ScanInfo(
+                ProductInfo(
+                    cve.vendor,
+                    cve.product,
+                    cve.version,
+                    cve.location,
+                    PackageURL(
+                        type="ad-hoc",
+                        namespace=cve.vendor,
+                        name=re.sub(r"[^a-zA-Z0-9._-]", "", cve.product).lower(),
+                        version=cve.version,
+                        qualifiers={},
+                        subpath=None,
+                    ),
+                ),
+                pathlib.Path(filename).resolve(),
+            )

cve_bin_tool/parsers/go.py

@@ -25,6 +25,10 @@ class GoParser(Parser):
 
     """
 
+    PARSER_MATCH_FILENAMES = [
+        "go.mod",
+    ]
+
     def __init__(self, cve_db, logger):
         super().__init__(cve_db, logger)
         self.purl_pkg_type = "golang"

cve_bin_tool/parsers/java.py

@@ -14,6 +14,10 @@
 class JavaParser(Parser):
     """Class to handle parsing Java-based Packages."""
 
+    PARSER_MATCH_FILENAMES = [
+        "pom.xml",
+    ]
+
     def __init__(self, cve_db, logger, validate=True):
         super().__init__(cve_db, logger)
         self.validate = validate

cve_bin_tool/parsers/javascript.py

@@ -11,6 +11,10 @@
 class JavascriptParser(Parser):
     """Parser for javascript's package-lock.json files"""
 
+    PARSER_MATCH_FILENAMES = [
+        "package-lock.json",
+    ]
+
     def __init__(self, cve_db, logger):
         super().__init__(cve_db, logger)
         self.purl_pkg_type = "npm"

cve_bin_tool/parsers/parse.py

@@ -1,40 +1,45 @@
 # Copyright (C) 2022 Intel Corporation
 # SPDX-License-Identifier: GPL-3.0-or-later
+from __future__ import annotations
 
-from cve_bin_tool.parsers.dart import DartParser
-from cve_bin_tool.parsers.go import GoParser
-from cve_bin_tool.parsers.java import JavaParser
-from cve_bin_tool.parsers.javascript import JavascriptParser
-from cve_bin_tool.parsers.perl import PerlParser
-from cve_bin_tool.parsers.php import PhpParser
-from cve_bin_tool.parsers.python import PythonParser, PythonRequirementsParser
-from cve_bin_tool.parsers.r import RParser
-from cve_bin_tool.parsers.ruby import RubyParser
-from cve_bin_tool.parsers.rust import RustParser
-from cve_bin_tool.parsers.swift import SwiftParser
-
-valid_files = {
-    "pom.xml": JavaParser,
-    "package-lock.json": JavascriptParser,
-    "Cargo.lock": RustParser,
-    "renv.lock": RParser,
-    "requirements.txt": PythonRequirementsParser,
-    "go.mod": GoParser,
-    "PKG-INFO: ": PythonParser,
-    "METADATA: ": PythonParser,
-    "Gemfile.lock": RubyParser,
-    "Package.resolved": SwiftParser,
-    "composer.lock": PhpParser,
-    "cpanfile": PerlParser,
-    "pubspec.lock": DartParser,
-}
+import sys
+
+if sys.version_info >= (3, 10):
+    from importlib import metadata as importlib_metadata
+else:
+    import importlib_metadata
+
+from cve_bin_tool.parsers import Parser
+
+PARSERS_ENTRYPOINT = "cve_bin_tool.parsers"
+
+
+def load_valid_files() -> dict[str, list[type[Parser]]]:
+    """Loads file parsers"""
+    valid_files: dict[str, list[type[Parser]]] = {}
+    for entrypoint in importlib_metadata.entry_points().select(
+        group=PARSERS_ENTRYPOINT
+    ):
+        parser_cls = entrypoint.load()
+        for match_filename in getattr(parser_cls, "PARSER_MATCH_FILENAMES", []):
+            valid_files.setdefault(match_filename, [])
+            valid_files[match_filename].append(parser_cls)
+    for match_filename in valid_files:
+        valid_files[match_filename] = list(set(valid_files[match_filename]))
+    return valid_files
+
+
+valid_files = load_valid_files()
 
 
 def parse(filename, output, cve_db, logger):
     """
     Parses the given filename using the appropriate parser.
     """
+    parsers = []
     for file in list(valid_files.keys()):
         if file in output:
-            parser = valid_files[file](cve_db, logger)
-    yield from parser.run_checker(filename)
+            for valid_file_parser in valid_files[file]:
+                parsers.append(valid_file_parser(cve_db, logger))
+    for parser in parsers:
+        yield from parser.run_checker(filename)

cve_bin_tool/parsers/perl.py

@@ -9,6 +9,10 @@
 class PerlParser(Parser):
     """Parser for perl's cpan files"""
 
+    PARSER_MATCH_FILENAMES = [
+        "cpanfile",
+    ]
+
     def __init__(self, cve_db, logger):
         super().__init__(cve_db, logger)
         self.purl_pkg_type = "cpan"

cve_bin_tool/parsers/php.py

@@ -17,6 +17,10 @@ class PhpParser(Parser):
     generate PURLs (Package URLs) for the listed packages.
     """
 
+    PARSER_MATCH_FILENAMES = [
+        "composer.lock",
+    ]
+
     def __init__(self, cve_db, logger):
         """Initialize the PhpParser."""
         super().__init__(cve_db, logger)