test: output if domain is in allowed domain list (#2776)

Author: mikhailswift

e2e/cluster/cmx/cluster.go

@@ -382,12 +382,17 @@ func (c *Cluster) SetupPlaywrightAndRunTest(testName string, args ...string) (st
 	return c.RunPlaywrightTest(testName, args...)
 }
 
-func (c *Cluster) SetupPlaywright(envs ...map[string]string) error {
+func (c *Cluster) BypassKurlProxy(envs ...map[string]string) error {
 	c.t.Logf("%s: bypassing kurl-proxy", time.Now().Format(time.RFC3339))
 	_, stderr, err := c.RunCommandOnNode(0, []string{"/usr/local/bin/bypass-kurl-proxy.sh"}, envs...)
 	if err != nil {
 		return fmt.Errorf("bypass kurl-proxy: %v: %s", err, string(stderr))
 	}
+
+	return nil
+}
+
+func (c *Cluster) NPMInstallPlaywright(envs ...map[string]string) error {
 	c.t.Logf("%s: installing playwright", time.Now().Format(time.RFC3339))
 	output, err := exec.Command("sh", "-c", "cd playwright && npm ci && npx playwright install --with-deps").CombinedOutput()
 	if err != nil {
@@ -396,6 +401,14 @@ func (c *Cluster) SetupPlaywright(envs ...map[string]string) error {
 	return nil
 }
 
+func (c *Cluster) SetupPlaywright(envs ...map[string]string) error {
+	if err := c.BypassKurlProxy(envs...); err != nil {
+		return err
+	}
+
+	return c.NPMInstallPlaywright(envs...)
+}
+
 func (c *Cluster) RunPlaywrightTest(testName string, args ...string) (string, string, error) {
 	c.t.Logf("%s: running playwright test %s", time.Now().Format(time.RFC3339), testName)
 	cmdArgs := []string{testName}
@@ -568,13 +581,12 @@ func (c *Cluster) waitUntilRunning(node Node, nodeNum int, timeoutDuration time.
 	}
 }
 
-func (c *Cluster) CollectNetworkReport() ([]NetworkEvent, []byte, error) {
+func (c *Cluster) CollectNetworkReport() ([]NetworkEvent, error) {
 	output, err := exec.Command("replicated", "network", "report", fmt.Sprintf("--id=%v", c.network.ID), "-ojson").Output()
 	if err != nil {
-		return nil, nil, fmt.Errorf("collect network report: %v", err)
+		return nil, fmt.Errorf("collect network report: %v", err)
 	}
 
-	// TODO: investigate CLI changes to make event_data a json object instead of a string
 	type eventWrapper struct {
 		EventData string `json:"event_data"`
 	}
@@ -585,18 +597,18 @@ func (c *Cluster) CollectNetworkReport() ([]NetworkEvent, []byte, error) {
 
 	report := networkReport{}
 	if err := json.Unmarshal(output, &report); err != nil {
-		return nil, nil, fmt.Errorf("unmarshal network events: %v", err)
+		return nil, fmt.Errorf("unmarshal network events: %v", err)
 	}
 
 	networkEvents := make([]NetworkEvent, 0, len(report.Events))
 	for _, e := range report.Events {
 		ne := NetworkEvent{}
 		if err := json.Unmarshal([]byte(e.EventData), &ne); err != nil {
-			return nil, nil, fmt.Errorf("unmarshal network event data: %v", err)
+			return nil, fmt.Errorf("unmarshal network event data: %v", err)
 		}
 
 		networkEvents = append(networkEvents, ne)
 	}
 
-	return networkEvents, output, nil
+	return networkEvents, nil
 }

e2e/install_test.go

@@ -3,6 +3,7 @@ package e2e
 import (
 	"encoding/base64"
 	"fmt"
+	"net"
 	"os"
 	"strings"
 	"testing"
@@ -2072,60 +2073,72 @@ func TestSingleNodeNetworkReport(t *testing.T) {
 	})
 	defer tc.Cleanup()
 
+	if err := tc.NPMInstallPlaywright(); err != nil {
+		t.Fatalf("fail to setup playwright: %v", err)
+	}
+
 	if err := tc.SetNetworkReport(true); err != nil {
 		t.Fatalf("failed to enable network reporting: %v", err)
 	}
 
 	downloadECRelease(t, tc, 0)
 	installSingleNode(t, tc)
-	if stdout, stderr, err := tc.SetupPlaywrightAndRunTest("deploy-app"); err != nil {
+
+	if err := tc.BypassKurlProxy(); err != nil {
+		t.Fatalf("fail to bypass kurl-proxy: %v", err)
+	}
+
+	if stdout, stderr, err := tc.RunPlaywrightTest("deploy-app"); err != nil {
 		t.Fatalf("fail to run playwright test deploy-app: %v: %s: %s", err, stdout, stderr)
 	}
 
 	checkInstallationState(t, tc)
 	checkNodeJoinCommand(t, tc, 0)
 
+	// TODO: network events can came a few seconds to flow from cluster-provisioner, should look into ways to signal when a report has finished
+	time.Sleep(20 * time.Second)
+
 	if err := tc.SetNetworkReport(false); err != nil {
 		t.Fatalf("failed to disable network reporting: %v", err)
 	}
 
-	// TODO: network events can came a few seconds to flow from cluster-provisioner, should look into ways to signal when a report has finished
-	time.Sleep(5 * time.Second)
-
-	networkEvents, _, err := tc.CollectNetworkReport()
+	networkEvents, err := tc.CollectNetworkReport()
 	if err != nil {
 		t.Fatalf("failed to collect network report: %v", err)
 	}
 
-	domainsByIps := make(map[string]map[string]struct{})
-	for _, ne := range networkEvents {
-		// filter out local traffic
-		if ne.DstIP == "0.0.0.0" {
-			continue
-		}
+	allowedDomains := map[string]struct{}{
+		"ec-e2e-proxy.testcluster.net":          {},
+		"ec-e2e-replicated-app.testcluster.net": {},
 
-		domains := domainsByIps[ne.DstIP]
-		if domains == nil {
-			domains = make(map[string]struct{})
-		}
+		// these two appear due to the install_cots_cli function in single-node-install.sh
+		"kots.io":                              {},
+		"release-assets.githubusercontent.com": {},
+	}
 
-		if len(strings.TrimSpace(ne.DNSQueryName)) > 0 {
-			domains[ne.DNSQueryName] = struct{}{}
+	seenAllowedDomains := map[string]struct{}{}
+	t.Log("Logged outbound external network accesses:")
+	for _, ne := range networkEvents {
+		if ne.DNSQueryName == "" {
+			continue
 		}
 
-		domainsByIps[ne.DstIP] = domains
-	}
-
-	t.Log("Logged outbound external network accesses:\n")
-	for ip, domains := range domainsByIps {
-		domainOutput := ""
-		for domain := range domains {
-			domainOutput += fmt.Sprintf("\t- %v\n", domain)
+		// TODO: currently cmx reporting will return an ip as a domain, remove this once fixed
+		if ip := net.ParseIP(ne.DNSQueryName); ip != nil {
+			continue
 		}
 
-		t.Logf("IP: %v", ip)
-		if len(domainOutput) > 0 {
-			t.Logf("\n%v", domainOutput)
+		_, allowed := allowedDomains[ne.DNSQueryName]
+		// only print allowed domains once to reduce test output noise, but print every violation we see
+		if allowed {
+			if _, ok := seenAllowedDomains[ne.DNSQueryName]; !ok {
+				t.Logf("%v - ALLOWED", ne.DNSQueryName)
+				seenAllowedDomains[ne.DNSQueryName] = struct{}{}
+			}
+		} else {
+			t.Logf("%v - UNALLOWED\n", ne.DNSQueryName)
+			t.Logf("\tUnallowed event details: %+v", ne)
+			t.Fail()
 		}
 	}
 }