From 23d02314fff33e9f46697d46a55381f104f5ff01 Mon Sep 17 00:00:00 2001 From: David Cassany Date: Wed, 11 Jul 2018 10:30:53 +0200 Subject: [PATCH 1/2] Add rpmlayer differ This commit adds the rpmlayer differ that allows to perform analysis at rpm package level on each layer. Lists new, deleted and modified (different version) packages on each layer. Signed-off-by: David Cassany --- cmd/root.go | 2 +- differs/differs.go | 1 + differs/rpm_diff.go | 120 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 119 insertions(+), 4 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index e2a949da..592c7aaa 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -59,7 +59,7 @@ const ( RemotePrefix = "remote://" ) -var layerAnalyzers = [...]string{"layer", "aptlayer"} +var layerAnalyzers = [...]string{"layer", "aptlayer", "rpmlayer"} var RootCmd = &cobra.Command{ Use: "container-diff", diff --git a/differs/differs.go b/differs/differs.go index c8c617b2..597851be 100644 --- a/differs/differs.go +++ b/differs/differs.go @@ -49,6 +49,7 @@ var Analyzers = map[string]Analyzer{ "apt": AptAnalyzer{}, "aptlayer": AptLayerAnalyzer{}, "rpm": RPMAnalyzer{}, + "rpmlayer": RPMLayerAnalyzer{}, "pip": PipAnalyzer{}, "node": NodeAnalyzer{}, } diff --git a/differs/rpm_diff.go b/differs/rpm_diff.go index cc39019a..3f97923c 100644 --- a/differs/rpm_diff.go +++ b/differs/rpm_diff.go @@ -34,6 +34,8 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/daemon" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util" "github.com/GoogleContainerTools/container-diff/util" @@ -101,7 +103,7 @@ func (a RPMAnalyzer) getPackages(image pkgutil.Image) (map[string]util.PackageIn packages, err := rpmDataFromImageFS(image) if err != nil { logrus.Info("Running RPM binary from image in a container") - return rpmDataFromContainer(image) + return rpmDataFromContainer(image.Image) } return packages, err } @@ -164,7 +166,7 @@ func rpmDBPath(rootFSPath string) (string, error) { // rpmDataFromContainer runs image in a container, queries the data of // installed rpm packages and returns a map of packages. -func rpmDataFromContainer(image pkgutil.Image) (map[string]util.PackageInfo, error) { +func rpmDataFromContainer(image v1.Image) (map[string]util.PackageInfo, error) { packages := make(map[string]util.PackageInfo) client, err := godocker.NewClientFromEnv() @@ -175,7 +177,7 @@ func rpmDataFromContainer(image pkgutil.Image) (map[string]util.PackageInfo, err return packages, err } - imageName, err := loadImageToDaemon(image.Image) + imageName, err := loadImageToDaemon(image) if err != nil { return packages, fmt.Errorf("Error loading image: %s", err) @@ -364,3 +366,115 @@ func unlock() error { daemonMutex.Unlock() return nil } + +type RPMLayerAnalyzer struct { +} + +// Name returns the name of the analyzer. +func (a RPMLayerAnalyzer) Name() string { + return "RPMLayerAnalyzer" +} + +// Diff compares the installed rpm packages of image1 and image2 for each layer +func (a RPMLayerAnalyzer) Diff(image1, image2 pkgutil.Image) (util.Result, error) { + diff, err := singleVersionLayerDiff(image1, image2, a) + return diff, err +} + +// Analyze collects information of the installed rpm packages on each layer +func (a RPMLayerAnalyzer) Analyze(image pkgutil.Image) (util.Result, error) { + analysis, err := singleVersionLayerAnalysis(image, a) + return analysis, err +} + +// getPackages returns an array of maps of installed rpm packages on each layer +func (a RPMLayerAnalyzer) getPackages(image pkgutil.Image) ([]map[string]util.PackageInfo, error) { + path := image.FSPath + var packages []map[string]util.PackageInfo + if _, err := os.Stat(path); err != nil { + // invalid image directory path + return packages, err + } + + // try to find the rpm binary in bin/ or usr/bin/ + rpmBinary := filepath.Join(path, "bin/rpm") + if _, err := os.Stat(rpmBinary); err != nil { + rpmBinary = filepath.Join(path, "usr/bin/rpm") + if _, err = os.Stat(rpmBinary); err != nil { + logrus.Errorf("Could not detect RPM binary in unpacked image %s", image.Source) + return packages, nil + } + } + + packages, err := rpmDataFromLayerFS(image) + if err != nil { + logrus.Info("Running RPM binary from image in a container") + return rpmDataFromLayeredContainers(image.Image) + } + return packages, err +} + +// rpmDataFromLayerFS runs a local rpm binary, if any, to query the layer +// rpmdb and returns an array of maps of installed packages. +func rpmDataFromLayerFS(image pkgutil.Image) ([]map[string]util.PackageInfo, error) { + var packages []map[string]util.PackageInfo + // Check there is an executable rpm tool in host + if err := exec.Command("rpm", "--version").Run(); err != nil { + logrus.Warn("No RPM binary in host") + return packages, err + } + dbPath, err := rpmDBPath(image.FSPath) + if err != nil { + logrus.Warnf("Couldn't find RPM database: %s", err.Error()) + return packages, err + } + for _, layer := range image.Layers { + layerPackages := make(map[string]util.PackageInfo) + //query only layers that include the rpm database + if _, err := os.Stat(filepath.Join(layer.FSPath, dbPath)); err == nil { + cmdArgs := append([]string{"--root", layer.FSPath, "--dbpath", dbPath}, rpmCmd[1:]...) + out, err := exec.Command(rpmCmd[0], cmdArgs...).Output() + if err != nil { + logrus.Warnf("RPM call failed: %s", err.Error()) + return packages, err + } + layerPackages, err = parsePackageData(strings.Split(string(out), "\n")) + if err != nil { + return packages, err + } + } + packages = append(packages, layerPackages) + } + + return packages, nil +} + +// rpmDataFromLayeredContainers runs a tmp image in a container for each layer, +// queries the data of installed rpm packages and returns an array of maps of +// packages. +func rpmDataFromLayeredContainers(image v1.Image) ([]map[string]util.PackageInfo, error) { + var packages []map[string]util.PackageInfo + tmpImage, err := random.Image(0, 0) + if err != nil { + return packages, err + } + layers, err := image.Layers() + if err != nil { + return packages, err + } + // Append layers one by one to an empty image and query rpm + // database on each iteration + for _, layer := range layers { + tmpImage, err = mutate.AppendLayers(tmpImage, layer) + if err != nil { + return packages, err + } + layerPackages, err := rpmDataFromContainer(tmpImage) + if err != nil { + return packages, err + } + packages = append(packages, layerPackages) + } + + return packages, nil +} From 6751a8c06e426ea3af0271a7a5240b0d2b1ae14b Mon Sep 17 00:00:00 2001 From: David Cassany Date: Thu, 2 Aug 2018 11:33:58 +0200 Subject: [PATCH 2/2] Improving coding style * Refactored RPMLayerAnalyzer for better code reuse * Updated differs.go to initialize Analyzers map with constant keys * moved layerAnalyzers vector from root.go to differs.go Signed-off-by: David Cassany --- cmd/root.go | 4 +-- differs/differs.go | 33 +++++++++++++------ differs/rpm_diff.go | 77 ++++++++++++++++++++++----------------------- 3 files changed, 61 insertions(+), 53 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 592c7aaa..e421e0e2 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -59,8 +59,6 @@ const ( RemotePrefix = "remote://" ) -var layerAnalyzers = [...]string{"layer", "aptlayer", "rpmlayer"} - var RootCmd = &cobra.Command{ Use: "container-diff", Short: "container-diff is a tool for analyzing and comparing container images", @@ -270,7 +268,7 @@ func getExtractPathForName(name string) (string, error) { func includeLayers() bool { for _, t := range types { - for _, a := range layerAnalyzers { + for _, a := range differs.LayerAnalyzers { if t == a { return true } diff --git a/differs/differs.go b/differs/differs.go index 597851be..92557e59 100644 --- a/differs/differs.go +++ b/differs/differs.go @@ -24,6 +24,17 @@ import ( "github.com/sirupsen/logrus" ) +const historyAnalyzer = "history" +const metadataAnalyzer = "metadata" +const fileAnalyzer = "file" +const layerAnalyzer = "layer" +const aptAnalyzer = "apt" +const aptLayerAnalyzer = "aptlayer" +const rpmAnalyzer = "rpm" +const rpmLayerAnalyzer = "rpmlayer" +const pipAnalyzer = "pip" +const nodeAnalyzer = "node" + type DiffRequest struct { Image1 pkgutil.Image Image2 pkgutil.Image @@ -42,18 +53,20 @@ type Analyzer interface { } var Analyzers = map[string]Analyzer{ - "history": HistoryAnalyzer{}, - "metadata": MetadataAnalyzer{}, - "file": FileAnalyzer{}, - "layer": FileLayerAnalyzer{}, - "apt": AptAnalyzer{}, - "aptlayer": AptLayerAnalyzer{}, - "rpm": RPMAnalyzer{}, - "rpmlayer": RPMLayerAnalyzer{}, - "pip": PipAnalyzer{}, - "node": NodeAnalyzer{}, + historyAnalyzer: HistoryAnalyzer{}, + metadataAnalyzer: MetadataAnalyzer{}, + fileAnalyzer: FileAnalyzer{}, + layerAnalyzer: FileLayerAnalyzer{}, + aptAnalyzer: AptAnalyzer{}, + aptLayerAnalyzer: AptLayerAnalyzer{}, + rpmAnalyzer: RPMAnalyzer{}, + rpmLayerAnalyzer: RPMLayerAnalyzer{}, + pipAnalyzer: PipAnalyzer{}, + nodeAnalyzer: NodeAnalyzer{}, } +var LayerAnalyzers = [...]string{layerAnalyzer, aptLayerAnalyzer, rpmLayerAnalyzer} + func (req DiffRequest) GetDiff() (map[string]util.Result, error) { img1 := req.Image1 img2 := req.Image2 diff --git a/differs/rpm_diff.go b/differs/rpm_diff.go index 3f97923c..dcd83d55 100644 --- a/differs/rpm_diff.go +++ b/differs/rpm_diff.go @@ -102,7 +102,7 @@ func (a RPMAnalyzer) getPackages(image pkgutil.Image) (map[string]util.PackageIn packages, err := rpmDataFromImageFS(image) if err != nil { - logrus.Info("Running RPM binary from image in a container") + logrus.Info("Couldn't retrieve RPM data from extracted filesystem; running query in container") return rpmDataFromContainer(image.Image) } return packages, err @@ -111,30 +111,22 @@ func (a RPMAnalyzer) getPackages(image pkgutil.Image) (map[string]util.PackageIn // rpmDataFromImageFS runs a local rpm binary, if any, to query the image // rpmdb and returns a map of installed packages. func rpmDataFromImageFS(image pkgutil.Image) (map[string]util.PackageInfo, error) { - packages := make(map[string]util.PackageInfo) - // Check there is an executable rpm tool in host - if err := exec.Command("rpm", "--version").Run(); err != nil { - logrus.Warn("No RPM binary in host") - return packages, err - } - dbPath, err := rpmDBPath(image.FSPath) + dbPath, err := rpmEnvCheck(image.FSPath) if err != nil { logrus.Warnf("Couldn't find RPM database: %s", err.Error()) - return packages, err - } - cmdArgs := append([]string{"--root", image.FSPath, "--dbpath", dbPath}, rpmCmd[1:]...) - out, err := exec.Command(rpmCmd[0], cmdArgs...).Output() - if err != nil { - logrus.Warnf("RPM call failed: %s", err.Error()) - return packages, err + return nil, err } - output := strings.Split(string(out), "\n") - return parsePackageData(output) + return rpmDataFromFS(image.FSPath, dbPath) } -// rpmDBPath tries to get the RPM database path from the /usr/lib/rpm/macros -// file in the image rootfs. -func rpmDBPath(rootFSPath string) (string, error) { +// rpmEnvCheck checks there is an rpm binary in the host and tries to +// get the RPM database path from the /usr/lib/rpm/macros file in the +// image rootfs +func rpmEnvCheck(rootFSPath string) (string, error) { + if err := exec.Command("rpm", "--version").Run(); err != nil { + logrus.Warn("No RPM binary in host") + return "", err + } imgMacrosFile, err := os.Open(filepath.Join(rootFSPath, rpmMacros)) if err != nil { return "", err @@ -408,7 +400,7 @@ func (a RPMLayerAnalyzer) getPackages(image pkgutil.Image) ([]map[string]util.Pa packages, err := rpmDataFromLayerFS(image) if err != nil { - logrus.Info("Running RPM binary from image in a container") + logrus.Info("Couldn't retrieve RPM data from extracted filesystem; running query in container") return rpmDataFromLayeredContainers(image.Image) } return packages, err @@ -418,30 +410,15 @@ func (a RPMLayerAnalyzer) getPackages(image pkgutil.Image) ([]map[string]util.Pa // rpmdb and returns an array of maps of installed packages. func rpmDataFromLayerFS(image pkgutil.Image) ([]map[string]util.PackageInfo, error) { var packages []map[string]util.PackageInfo - // Check there is an executable rpm tool in host - if err := exec.Command("rpm", "--version").Run(); err != nil { - logrus.Warn("No RPM binary in host") - return packages, err - } - dbPath, err := rpmDBPath(image.FSPath) + dbPath, err := rpmEnvCheck(image.FSPath) if err != nil { logrus.Warnf("Couldn't find RPM database: %s", err.Error()) return packages, err } for _, layer := range image.Layers { - layerPackages := make(map[string]util.PackageInfo) - //query only layers that include the rpm database - if _, err := os.Stat(filepath.Join(layer.FSPath, dbPath)); err == nil { - cmdArgs := append([]string{"--root", layer.FSPath, "--dbpath", dbPath}, rpmCmd[1:]...) - out, err := exec.Command(rpmCmd[0], cmdArgs...).Output() - if err != nil { - logrus.Warnf("RPM call failed: %s", err.Error()) - return packages, err - } - layerPackages, err = parsePackageData(strings.Split(string(out), "\n")) - if err != nil { - return packages, err - } + layerPackages, err := rpmDataFromFS(layer.FSPath, dbPath) + if err != nil { + return packages, err } packages = append(packages, layerPackages) } @@ -449,6 +426,26 @@ func rpmDataFromLayerFS(image pkgutil.Image) ([]map[string]util.PackageInfo, err return packages, nil } +// rpmDataFromFS runs a local rpm binary to query the image +// rpmdb and returns a map of installed packages. +func rpmDataFromFS(fsPath string, dbPath string) (map[string]util.PackageInfo, error) { + packages := make(map[string]util.PackageInfo) + if _, err := os.Stat(filepath.Join(fsPath, dbPath)); err == nil { + cmdArgs := append([]string{"--root", fsPath, "--dbpath", dbPath}, rpmCmd[1:]...) + out, err := exec.Command(rpmCmd[0], cmdArgs...).Output() + if err != nil { + logrus.Warnf("RPM call failed: %s", err.Error()) + return packages, err + } + output := strings.Split(string(out), "\n") + packages, err := parsePackageData(output) + if err != nil { + return packages, err + } + } + return packages, nil +} + // rpmDataFromLayeredContainers runs a tmp image in a container for each layer, // queries the data of installed rpm packages and returns an array of maps of // packages.