Update existing lists

bsyk · bsyk · commit b7b3a5a50d28 · 2025-03-30T13:25:27.000-07:00
To avoid the need to delete all lists and recreate them, we can update existing lists only when their contents had changed.
This processes the diffs between the desired list of domains and the existing lists. Removing entires that are no longer in the desired lists and appending any new entries. This prefers to minimize the number of PATCH calls by appending entries to the lists we're already patching for the removals.
The priority for additions is:
1. Add to lists we're already patching for removals.
2. Add to existing lists with fewer than LIST_ITEM_SIZE entries.
3. Create a new list.
diff --git a/cf_gateway_rule_create.js b/cf_gateway_rule_create.js
@@ -11,7 +11,7 @@ const wirefilterDNSExpression = lists.reduce((previous, current) => {
   return `${previous} any(dns.domains[*] in \$${current.id}) or `;
 }, "");
 
-console.log("Creating DNS rule...");
+console.log("Checking DNS rule...");
 // .slice removes the trailing ' or '
 await upsertZeroTrustRule(wirefilterDNSExpression.slice(0, -4), "CGPS Filter Lists", ["dns"]);
 
diff --git a/cf_list_create.js b/cf_list_create.js
@@ -1,7 +1,7 @@
 import { existsSync } from "node:fs";
 import { resolve } from "node:path";
 
-import { createZeroTrustListsOneByOne } from "./lib/api.js";
+import { synchronizeZeroTrustLists } from "./lib/api.js";
 import {
   DEBUG,
   DRY_RUN,
@@ -140,7 +140,7 @@ console.log("\n\n");
     `Creating ${numberOfLists} lists for ${domains.length} domains...`
   );
 
-  await createZeroTrustListsOneByOne(domains);
+  await synchronizeZeroTrustLists(domains);
   await notifyWebhook(
     `CF List Create script finished running (${domains.length} domains, ${numberOfLists} lists)`
   );
diff --git a/lib/api.js b/lib/api.js
@@ -12,6 +12,19 @@ export const getZeroTrustLists = () =>
     method: "GET",
   });
 
+/**
+ * Gets Zero Trust list items
+ *
+ * API docs: https://developers.cloudflare.com/api/operations/zero-trust-lists-zero-trust-list-items
+ * @param {string} id The id of the list.
+ * @returns {Promise<Object>}
+ */
+const getZeroTrustListItems = (id) =>
+  requestGateway(`/lists/${id}/items?per_page=${LIST_ITEM_SIZE}`, {
+    method: "GET",
+  });
+
+
 /**
  * Creates a Zero Trust list.
  *
@@ -31,14 +44,141 @@ const createZeroTrustList = (name, items) =>
     }),
   });
 
+/**
+ * Patches an existing list. Remove/append entries to the list.
+ *
+ * API docs: https://developers.cloudflare.com/api/operations/zero-trust-lists-patch-zero-trust-list
+ * @param {string} listId The ID of the list to patch
+ * @param {Object} patch The changes to make
+ * @param {string[]} patch.remove A list of the item values you want to remove.
+ * @param {Object[]} patch.append Items to add to the list.
+ * @param {string} patch.append[].value The domain of an entry.
+ * @returns
+ */
+const patchExistingList = (listId, patch) =>
+  requestGateway(`/lists/${listId}`, {
+    method: "PATCH",
+    body: JSON.stringify(patch),
+  });
+
+/**
+ * Synchronize Zero Trust lists.
+ * Inspects existing lists starting with "CGPS List"
+ * Compares the entries in the lists with the desired domains in the items.
+ * Removes any entries in the lists that are not in the items.
+ * Adds any entries that are in the items and not in the lists.
+ * Uses available capacity in existing lists prior to creating a new list.
+ * @param {string[]} items The domains.
+ */
+export const synchronizeZeroTrustLists = async (items) => {
+  const itemSet = new Set(items);
+
+  console.log("Checking existing lists...");
+  const { result: lists } = await getZeroTrustLists();
+  const cgpsLists = lists?.filter(({ name }) => name.startsWith("CGPS List")) || [];
+  console.log(`Found ${cgpsLists.length} existing lists. Calculating diffs...`);
+
+  const domainsByList = {};
+  // Do this sequentially to avoid rate-limits
+  for (const list of cgpsLists) {
+    const { result: listItems, result_info } = await getZeroTrustListItems(list.id);
+    if (result_info.total_count > LIST_ITEM_SIZE) {
+      console.log(`List ${list.name} contains more entries that LIST_ITEM_SIZE. Checking only the first ${LIST_ITEM_SIZE} entires. You may want to delete this list and recreate using the same size limit.`);
+    }
+    domainsByList[list.id] = listItems?.map(item => item.value) || [];
+  }
+
+  // Extract all the list entries into a map, keyed by domain, pointing to the list.
+  const existingDomains = Object.fromEntries(
+    Object.entries(domainsByList).flatMap(([id, domains]) => domains.map(d => [d, id]))
+  );
+
+  // Create a list of entries to remove.
+  // Iterate the existing list(s) removing anything that's in the new list.
+  // Resulting in entries that are in the existing list(s) and not in the new list.
+  const toRemove = Object.fromEntries(
+    Object.entries(existingDomains).filter(([domain]) => !itemSet.has(domain))
+  );
+
+  // Create a list of entries to add.
+  // Iterate the new list keeping only entries not in the existing list(s).
+  // Resulting in entries that need to be added.
+  const toAdd = items.filter(domain => !existingDomains[domain]);
+
+  console.log(`${Object.keys(toRemove).length} removals, ${toAdd.length} additions to make`);
+
+  // Group the removals by list id, so we can make a patch request.
+  const removalPatches = Object.entries(toRemove).reduce((acc, [domain, listId]) => {
+    acc[listId] = acc[listId] || { remove: [] };
+    acc[listId].remove.push(domain);
+    return acc;
+  }, {});
+
+  // Fill any "gaps" in the lists made by the removals with any additions.
+  // If we can fit all the additions into the same lists that we're processing removals
+  // we can minimize the number of lists that need to be edited.
+  const patches = Object.fromEntries(
+    Object.entries(removalPatches).map(([listId, patch]) => {
+      // Work out how much "space" is in the list by looking at
+      // how many entries there were and how many we're removing.
+      const spaceInList = LIST_ITEM_SIZE - (domainsByList[listId].length - patch.remove.length);
+      // Take upto spaceInList entries from the additions into this list.
+      const append = Array(spaceInList)
+        .fill(0)
+        .map(() => toAdd.shift())
+        .filter(Boolean)
+        .map(domain => ({ value: domain }));
+      return [listId, { ...patch, append }];
+    })
+  );
+
+  // Are there any more appends remaining?
+  if (toAdd.length) {
+    // Is there any space in any existing lists, other than those we're already patching?
+    const unpatchedListIds = Object.keys(domainsByList).filter(listId => !patches[listId]);
+    unpatchedListIds.forEach(listId => {
+      const spaceInList = LIST_ITEM_SIZE - domainsByList[listId].length;
+      if (spaceInList > 0) {
+        // Take upto spaceInList entries from the additions into this list.
+        const append = Array(spaceInList)
+          .fill(0)
+          .map(() => toAdd.shift())
+          .filter(Boolean)
+          .map(domain => ({ value: domain }));
+
+        // Add this list edit to the patches
+        if (append.length) {
+          patches[listId] = { append };
+        }
+      }
+    });
+  }
+
+  // Process all the patches. Sequentially to avoid rate limits.
+  for(const [listId, patch] of Object.entries(patches)) {
+    const appends = !!patch.append ? patch.append.length : 0;
+    const removals = !!patch.remove ? patch.remove.length : 0;
+    console.log(`Updating list "${cgpsLists.find(list => list.id === listId).name}"${appends ? `, ${appends} additions` : ''}${removals ? `, ${removals} removals` : ''}`);
+    await patchExistingList(listId, patch);
+  }
+
+  // Are there any more appends remaining?
+  if (toAdd.length) {
+    // We'll need to create new list(s)
+    const nextListNumber = Math.max(0, ...cgpsLists.map(list => parseInt(list.name.replace('CGPS List - Chunk ', ''))).filter(x => Number.isInteger(x))) + 1;
+    await createZeroTrustListsOneByOne(toAdd, nextListNumber);
+  }
+};
+
 /**
  * Creates Zero Trust lists sequentially.
  * @param {string[]} items The domains.
+ * @param {Number} [startingListNumber] The chunk number to start from when naming lists.
  */
-export const createZeroTrustListsOneByOne = async (items) => {
+export const createZeroTrustListsOneByOne = async (items, startingListNumber = 1) => {
   let totalListNumber = Math.ceil(items.length / LIST_ITEM_SIZE);
 
-  for (let i = 0, listNumber = 1; i < items.length; i += LIST_ITEM_SIZE) {
+  for (let i = 0, listNumber = startingListNumber; i < items.length; i += LIST_ITEM_SIZE) {
     const chunk = items
       .slice(i, i + LIST_ITEM_SIZE)
       .map((item) => ({ value: item }));
@@ -153,9 +293,9 @@ export const createZeroTrustRule = async (wirefilterExpression, name = "CGPS Fil
  * Updates a Zero Trust rule.
  *
  * API docs: https://developers.cloudflare.com/api/operations/zero-trust-gateway-rules-update-zero-trust-gateway-rule
- * @param {number} id The ID of the rule to be updated. 
+ * @param {number} id The ID of the rule to be updated.
  * @param {string} wirefilterExpression The expression to be used for the rule.
- * @param {string} name The name of the rule. 
+ * @param {string} name The name of the rule.
  * @param {string[]} filters The filters to be used for the rule.
  * @returns {Promise<Object>}
  */
diff --git a/package.json b/package.json
@@ -5,6 +5,8 @@
     "download": "node download_lists.js",
     "download:allowlist": "node download_lists.js allowlist",
     "download:blocklist": "node download_lists.js blocklist",
+    "cloudflare-refresh": "npm run download && npm run cloudflare-create",
+    "cloudflare-refresh:blocklist": "npm run download:blocklist && npm run cloudflare-create",
     "cloudflare-create": "npm run cloudflare-create:list && npm run cloudflare-create:rule",
     "cloudflare-delete": "npm run cloudflare-delete:rule && npm run cloudflare-delete:list",
     "cloudflare-create:rule": "node cf_gateway_rule_create.js",
