From 6294646827dae3a7bf3f9db408ae5604dd18c636 Mon Sep 17 00:00:00 2001 From: "John L. Singleton" Date: Wed, 14 Feb 2024 16:24:09 -0500 Subject: [PATCH] fix windows compatability; add flag for skipping compliation. --- codeql_bundle/cli.py | 4 ++++ codeql_bundle/helpers/bundle.py | 34 +++++++++++++++++++++++++-------- codeql_bundle/helpers/codeql.py | 22 +++++++++++++++++++++ 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/codeql_bundle/cli.py b/codeql_bundle/cli.py index 301403d..c5dc21a 100644 --- a/codeql_bundle/cli.py +++ b/codeql_bundle/cli.py @@ -40,6 +40,7 @@ type=click.Path(exists=True, path_type=Path), default=Path.cwd(), ) +@click.option('--no-precompile', '-nc', is_flag=True, help="Do not pre-compile the bundle.") @click.option( "-l", "--log", @@ -56,6 +57,7 @@ def main( bundle_path: Path, output: Path, workspace: Path, + no_precompile: bool, loglevel: str, platform: List[str], code_scanning_config: Optional[Path], @@ -82,6 +84,8 @@ def main( try: bundle = CustomBundle(bundle_path, workspace) + # options for custom bundle + bundle.disable_precompilation = no_precompile unsupported_platforms = list(filter(lambda p: not bundle.supports_platform(BundlePlatform.from_string(p)), platform)) if len(unsupported_platforms) > 0: diff --git a/codeql_bundle/helpers/bundle.py b/codeql_bundle/helpers/bundle.py index e0ca889..51521eb 100644 --- a/codeql_bundle/helpers/bundle.py +++ b/codeql_bundle/helpers/bundle.py @@ -144,6 +144,8 @@ def __str__(self): class Bundle: def __init__(self, bundle_path: Path) -> None: self.tmp_dir = TemporaryDirectory() + self.disable_precompilation = False + if bundle_path.is_dir(): self.bundle_path = Path(self.tmp_dir.name) / bundle_path.name shutil.copytree( @@ -190,7 +192,8 @@ def supports_windows() -> set[BundlePlatform]: elif current_system == "Windows" and BundlePlatform.WINDOWS not in self.platforms: raise BundleException("Bundle doesn't support Windows!") - self.codeql = CodeQL(self.bundle_path / "codeql") + self.codeql = CodeQL(self.bundle_codeql_exe) + try: logging.info(f"Validating the CodeQL CLI version part of the bundle.") unpacked_location = self.codeql.unpacked_location() @@ -215,12 +218,28 @@ def __del__(self) -> None: f"Removing temporary directory {self.tmp_dir.name} used to build custom bundle." ) self.tmp_dir.cleanup() - + def get_bundle_packs(self) -> List[ResolvedCodeQLPack]: return self.bundle_packs def supports_platform(self, platform: BundlePlatform) -> bool: return platform in self.platforms + + @property + def bundle_codeql_exe(self): + if platform.system() == "Windows": + return self.bundle_path / "codeql.exe" + + return self.bundle_path / "codeql" + + @property + def disable_precompilation(self): + return self._disable_precompilation + + @disable_precompilation.setter + def disable_precompilation(self, value: bool): + self._disable_precompilation = value + class CustomBundle(Bundle): def __init__(self, bundle_path: Path, workspace_path: Path = Path.cwd()) -> None: @@ -341,7 +360,7 @@ def bundle_customization_pack(customization_pack: ResolvedCodeQLPack): f"Bundling the customization pack {customization_pack_copy.config.name} at {customization_pack_copy.path}" ) self.codeql.pack_bundle( - customization_pack_copy, self.bundle_path / "qlpacks" + customization_pack_copy, self.bundle_path / "qlpacks", disable_precompilation=self.disable_precompilation ) def copy_pack(pack: ResolvedCodeQLPack) -> ResolvedCodeQLPack: @@ -470,13 +489,14 @@ def bundle_stdlib_pack(pack: ResolvedCodeQLPack): logging.debug( f"Bundling the standard library pack {pack_copy.config.name} at {pack_copy.path}" ) - self.codeql.pack_bundle(pack_copy, self.bundle_path / "qlpacks") + self.codeql.pack_bundle(pack_copy, self.bundle_path / "qlpacks", disable_precompilation=self.disable_precompilation) def bundle_library_pack(library_pack: ResolvedCodeQLPack): logging.info(f"Bundling the library pack {library_pack.config.name}.") self.codeql.pack_bundle( library_pack, self.bundle_path / "qlpacks", + disable_precompilation=self.disable_precompilation ) def bundle_query_pack(pack: ResolvedCodeQLPack): @@ -520,8 +540,7 @@ def bundle_query_pack(pack: ResolvedCodeQLPack): self.codeql.pack_create( pack_copy, self.bundle_path / "qlpacks", - self.bundle_path, - ) + self.bundle_path) else: logging.info(f"Bundling the query pack {pack.config.name}.") pack_copy = copy_pack(pack) @@ -537,8 +556,7 @@ def bundle_query_pack(pack: ResolvedCodeQLPack): self.codeql.pack_create( pack_copy, - self.bundle_path / "qlpacks", - ) + self.bundle_path / "qlpacks") sorted_packs = list(pack_sorter.static_order()) logger.debug(f"Sorted packs: {' -> '.join(map(lambda p: p.config.name, sorted_packs))}") diff --git a/codeql_bundle/helpers/codeql.py b/codeql_bundle/helpers/codeql.py index 085d2a5..f5a9462 100644 --- a/codeql_bundle/helpers/codeql.py +++ b/codeql_bundle/helpers/codeql.py @@ -65,6 +65,14 @@ def __init__(self, codeql_path: Path): self.codeql_path = codeql_path self._version = None + @property + def disable_precompilation(self): + return self._disable_precompilation + + @disable_precompilation.setter + def disable_precompilation(self, value: bool): + self._disable_precompilation = value + def _exec(self, command: str, *args: str) -> subprocess.CompletedProcess[str]: logger.debug( f"Running CodeQL command: {command} with arguments: {' '.join(args)}" @@ -124,11 +132,18 @@ def pack_bundle( pack: CodeQLPack, output_path: Path, *additional_packs: Path, + disable_precompilation = False ): if not pack.config.library: raise CodeQLException(f"Cannot bundle non-library pack {pack.config.name}!") args = ["bundle", "--format=json", f"--pack-path={output_path}"] + if disable_precompilation: + args.append("--no-precompile") + logging.warn( + f"NOTE: Precompilation is disabled for {pack.config.name}! This may result in slower query execution." + ) + if len(additional_packs) > 0: args.append(f"--additional-packs={':'.join(map(str,additional_packs))}") cp = self._exec( @@ -146,11 +161,18 @@ def pack_create( pack: CodeQLPack, output_path: Path, *additional_packs: Path, + disable_precompilation = False ): if pack.config.library: raise CodeQLException(f"Cannot bundle non-query pack {pack.config.name}!") args = ["create", "--format=json", f"--output={output_path}", "--threads=0", "--no-default-compilation-cache"] + if disable_precompilation: + args.append("--no-precompile") + logging.warn( + f"NOTE: Precompilation is disabled for {pack.config.name}! This may result in slower query execution." + ) + if self.supports_qlx(): args.append("--qlx") if len(additional_packs) > 0: