Address bundling order issue

The order in which packs are bundled didn't consider dependencies between packs. This could lead to failures when a bundle being added to the bundle couldn't resolve its dependencies from the bundle because they weren't added yet.

This fix introduces a dependency graph that orders the workspace packs and impacted bundle packs.

Author: rvermeulen

codeql_bundle/cli.py

@@ -1,3 +1,12 @@
+# Add the parent directory to the path if this module is run directly (i.e. not imported)
+# This is necessary to support both the Poetry script invocation and the direct invocation.
+if not __package__ and __name__ == "__main__":
+    import sys
+    from pathlib import Path
+
+    sys.path.append(str(Path(__file__).parent.parent))
+    __package__ = Path(__file__).parent.name
+
 import click
 from pathlib import Path
 from codeql_bundle.helpers.codeql import CodeQLException
@@ -8,7 +17,6 @@
 
 logger = logging.getLogger(__name__)
 
-
 @click.command()
 @click.option(
     "-b",
@@ -71,32 +79,32 @@ def main(
     try:
         bundle = CustomBundle(bundle_path, workspace)
         logger.info(f"Looking for CodeQL packs in workspace {workspace}")
-        packs_in_workspace = bundle.codeql.pack_ls(workspace)
+        packs_in_workspace = bundle.getCodeQLPacks()
         logger.info(
-            f"Found the CodeQL packs: {','.join(map(lambda p: p.name, packs_in_workspace))}"
+            f"Found the CodeQL packs: {','.join(map(lambda p: p.config.name, packs_in_workspace))}"
         )
 
         if len(packs) > 0:
             selected_packs = [
                 available_pack
                 for available_pack in packs_in_workspace
-                if available_pack.name in packs
+                if available_pack.config.name in packs
             ]
         else:
             selected_packs = packs_in_workspace
 
         logger.info(
-            f"Considering the following CodeQL packs for inclusion in the custom bundle: {','.join(map(lambda p: p.name, selected_packs))}"
+            f"Considering the following CodeQL packs for inclusion in the custom bundle: {','.join(map(lambda p: p.config.name, selected_packs))}"
         )
-        missing_packs = set(packs) - {pack.name for pack in selected_packs}
+        missing_packs = set(packs) - {pack.config.name for pack in selected_packs}
         if len(missing_packs) > 0:
             logger.fatal(
                 f"The provided CodeQL workspace doesn't contain the provided packs '{','.join(missing_packs)}'",
             )
             sys.exit(1)
 
         logger.info(
-            f"Adding the packs {','.join(map(lambda p: p.name, selected_packs))} to the custom bundle."
+            f"Adding the packs {','.join(map(lambda p: p.config.name, selected_packs))} and its workspace dependencies to the custom bundle."
         )
         bundle.add_packs(*selected_packs)
         logger.info(f"Bundling custom bundle at {output}")

codeql_bundle/helpers/bundle.py

@@ -5,7 +5,6 @@
 from typing import Dict, Any, Iterable, Self, Optional, List
 import yaml
 from dataclasses import dataclass, fields, field
-from enum import Enum
 import logging
 
 logger = logging.getLogger(__name__)
@@ -16,7 +15,7 @@ class CodeQLException(Exception):
 
 
 @dataclass(kw_only=True, frozen=True, eq=True)
-class CodeQLPack:
+class CodeQLPackConfig:
     library: bool = False
     name: str
     version: Version = Version("0.0.0")
@@ -53,25 +52,18 @@ def get_pack_name(self) -> str:
     def __hash__(self):
         return hash(f"{self.name}@{str(self.version)}")
 
-
-class CodeQLPackKind(Enum):
-    QUERY_PACK = 1
-    LIBRARY_PACK = 2
-    CUSTOMIZATION_PACK = 3
-
-
 @dataclass(kw_only=True, frozen=True, eq=True)
-class ResolvedCodeQLPack(CodeQLPack):
+class CodeQLPack:
     path: Path
-    kind: CodeQLPackKind
-
-    def __hash__(self):
-        return CodeQLPack.__hash__(self)
+    config: CodeQLPackConfig
 
+    def __hash__(self) -> int:
+        return hash(f"{self.path}")
 
 class CodeQL:
     def __init__(self, codeql_path: Path):
         self.codeql_path = codeql_path
+        self._version = None
 
     def _exec(self, command: str, *args: str) -> subprocess.CompletedProcess[str]:
         logger.debug(
@@ -80,16 +72,20 @@ def _exec(self, command: str, *args: str) -> subprocess.CompletedProcess[str]:
         return subprocess.run(
             [f"{self.codeql_path}", command] + [arg for arg in args],
             capture_output=True,
-            text=True,
+            text=True
         )
 
     def version(self) -> Version:
-        cp = self._exec("version", "--format=json")
-        if cp.returncode == 0:
-            version_info = json.loads(cp.stdout)
-            return Version(version_info["version"])
+        if self._version != None:
+            return self._version
         else:
-            raise CodeQLException(f"Failed to run {cp.args} command!")
+            cp = self._exec("version", "--format=json")
+            if cp.returncode == 0:
+                version_info = json.loads(cp.stdout)
+                self._version = Version(version_info["version"])
+                return self._version
+            else:
+                raise CodeQLException(f"Failed to run {cp.args} command!")
 
     def unpacked_location(self) -> Path:
         cp = self._exec("version", "--format=json")
@@ -102,44 +98,36 @@ def unpacked_location(self) -> Path:
     def supports_qlx(self) -> bool:
         return self.version() >= Version("2.11.4")
 
-    def pack_ls(self, workspace: Path = Path.cwd()) -> List[ResolvedCodeQLPack]:
+    def pack_ls(self, workspace: Path = Path.cwd()) -> List[CodeQLPack]:
         cp = self._exec("pack", "ls", "--format=json", str(workspace))
         if cp.returncode == 0:
             packs: Iterable[Path] = map(Path, json.loads(cp.stdout)["packs"].keys())
 
-            def load(qlpack: Path) -> ResolvedCodeQLPack:
-                with qlpack.open("r") as qlpack_file:
-                    logger.debug(f"Resolving CodeQL pack at {qlpack}.")
-                    qlpack_spec = yaml.safe_load(qlpack_file)
-                    qlpack_spec["path"] = qlpack
-                    if not "library" in qlpack_spec or not qlpack_spec["library"]:
-                        qlpack_spec["kind"] = CodeQLPackKind.QUERY_PACK
-                    else:
-                        if (
-                            qlpack_spec["path"].parent
-                            / qlpack_spec["name"].replace("-", "_")
-                            / "Customizations.qll"
-                        ).exists():
-                            qlpack_spec["kind"] = CodeQLPackKind.CUSTOMIZATION_PACK
-                        else:
-                            qlpack_spec["kind"] = CodeQLPackKind.LIBRARY_PACK
-                    resolved_pack = ResolvedCodeQLPack.from_dict(qlpack_spec)
+            def load(qlpack_yml_path: Path) -> CodeQLPack:
+                with qlpack_yml_path.open("r") as qlpack_yml_file:
+                    logger.debug(f"Loading CodeQL pack configuration at {qlpack_yml_path}.")
+                    qlpack_yml_as_dict: Dict[str, Any] = yaml.safe_load(qlpack_yml_file)
+                    qlpack_config = CodeQLPackConfig.from_dict(qlpack_yml_as_dict)
+                    qlpack = CodeQLPack(path=qlpack_yml_path, config=qlpack_config)
                     logger.debug(
-                        f"Resolved {resolved_pack.name} with version {str(resolved_pack.version)} at {resolved_pack.path} with kind {resolved_pack.kind.name}"
+                        f"Loaded {qlpack.config.name} with version {str(qlpack.config.version)} at {qlpack.path}."
                     )
-                    return resolved_pack
+                    return qlpack
 
-            logger.debug(f"Resolving CodeQL packs for workspace {workspace}")
+            logger.debug(f"Listing CodeQL packs for workspace {workspace}")
             return list(map(load, packs))
         else:
             raise CodeQLException(f"Failed to run {cp.args} command! {cp.stderr}")
 
     def pack_bundle(
         self,
-        pack: ResolvedCodeQLPack,
+        pack: CodeQLPack,
         output_path: Path,
         *additional_packs: Path,
     ):
+        if not pack.config.library:
+            raise CodeQLException(f"Cannot bundle non-library pack {pack.config.name}!")
+
         args = ["bundle", "--format=json", f"--pack-path={output_path}"]
         if len(additional_packs) > 0:
             args.append(f"--additional-packs={':'.join(map(str,additional_packs))}")
@@ -155,11 +143,14 @@ def pack_bundle(
 
     def pack_create(
         self,
-        pack: ResolvedCodeQLPack,
+        pack: CodeQLPack,
         output_path: Path,
         *additional_packs: Path,
     ):
-        args = ["create", "--format=json", f"--output={output_path}", "--threads=0"]
+        if pack.config.library:
+            raise CodeQLException(f"Cannot bundle non-query pack {pack.config.name}!")
+
+        args = ["create", "--format=json", f"--output={output_path}", "--threads=0", "--no-default-compilation-cache"]
         if self.supports_qlx():
             args.append("--qlx")
         if len(additional_packs) > 0:

codeql_bundle/helpers/codeql.py

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "codeql-bundle"
-version = "0.1.6"
+version = "0.1.7"
 description = "Tool to create custom CodeQL bundles"
 authors = ["Remco Vermeulen <[email protected]>"]
 readme = "README.md"

pyproject.toml

@@ -0,0 +1,2 @@
+provide:
+  - "**/qlpack.yml"

tests/workspace-with-cyclic-dep/codeql-workspace.yml

@@ -0,0 +1,7 @@
+---
+library: false
+warnOnImplicitThis: false
+name: cycle/x
+version: 0.0.1
+dependencies:
+  "cycle/y": "*"

tests/workspace-with-cyclic-dep/x/qlpack.yml

@@ -0,0 +1,7 @@
+---
+library: false
+warnOnImplicitThis: false
+name: cycle/y
+version: 0.0.1
+dependencies:
+  "cycle/z": "*"

tests/workspace-with-cyclic-dep/y/qlpack.yml

@@ -0,0 +1,7 @@
+---
+library: false
+warnOnImplicitThis: false
+name: cycle/z
+version: 0.0.1
+dependencies:
+  "cycle/x": "*"

tests/workspace-with-cyclic-dep/z/qlpack.yml

@@ -0,0 +1,2 @@
+provide:
+  - "**/qlpack.yml"

tests/workspace/codeql-workspace.yml

@@ -0,0 +1,12 @@
+---
+lockVersion: 1.0.0
+dependencies:
+  codeql/cpp-all:
+    version: 0.7.3
+  codeql/ssa:
+    version: 0.0.18
+  codeql/tutorial:
+    version: 0.0.11
+  codeql/util:
+    version: 0.0.11
+compiled: false