From 27d3353655ccfee2a343795897561071ef527bd8 Mon Sep 17 00:00:00 2001 From: max-ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:14:11 +0200 Subject: [PATCH 01/10] conditional post requests + test --- lib/helper.js | 78 ++++++++++++++---------- lib/webpagetest.js | 132 +++++++++++++++++++++++++++++++++++++--- test/edge-cases-test.js | 12 ++++ 3 files changed, 182 insertions(+), 40 deletions(-) diff --git a/lib/helper.js b/lib/helper.js index f9a67ce..739aeb3 100644 --- a/lib/helper.js +++ b/lib/helper.js @@ -5,24 +5,24 @@ * Released under the MIT License */ -var xml2js = require('xml2js'), - url = require('url'), - os = require('os'), - csv = require('csv'), - entities = require('entities'); +var xml2js = require('xml2js'), + url = require('url'), + os = require('os'), + csv = require('csv'), + entities = require('entities'); -var parser = new xml2js.Parser({explicitArray: false, mergeAttrs: true}); +var parser = new xml2js.Parser({ explicitArray: false, mergeAttrs: true }); var reNumber = /^[\.\+\-]?[\d\.]+$/, - reInvalidDec = /(?:\.\d*){2,}/, - reDec = /\./, - reLineBreak = /[\n\r]+/g, - reLastBreak = /\n$/, - reProtocol = /^https?:\/\//i, - reIp = /\d+\.\d+\.\d+\.\d+/, // 127.0.0.1 + reInvalidDec = /(?:\.\d*){2,}/, + reDec = /\./, + reLineBreak = /[\n\r]+/g, + reLastBreak = /\n$/, + reProtocol = /^https?:\/\//i, + reIp = /\d+\.\d+\.\d+\.\d+/, // 127.0.0.1 - TAB = '\t', - NEWLINE = '\n'; + TAB = '\t', + NEWLINE = '\n'; function parseNumber(s) { if (typeof s !== 'string' || !reNumber.test(s) || reInvalidDec.test(s)) { @@ -34,7 +34,7 @@ function parseNumber(s) { function normalizeObj(root) { if (typeof root === 'object') { - Object.keys(root).forEach(function(key) { + Object.keys(root).forEach(function (key) { var value = root[key]; if (typeof value === 'string') { if (value.length === 0 || value === '\n') { @@ -62,8 +62,8 @@ function xmlToObj(xml, callback) { function svToObj(delimiter, headers, sv) { var data, - start = 0, - obj = {}; + start = 0, + obj = {}; delimiter = delimiter || ','; @@ -101,11 +101,11 @@ function svToObj(delimiter, headers, sv) { } function csvParser(data, callback) { - csv.parse(data.toString(), { columns: true }, function(err, data) { + csv.parse(data.toString(), { columns: true }, function (err, data) { if (err) { callback.bind(this, err); } - csv.transform(data, function(row) { + csv.transform(data, function (row) { var key, value; for (key in row) { value = row[key].replace(/|<\/b>/g, ''); @@ -155,19 +155,33 @@ function scriptToString(data) { } // Build the RESTful API url call only -function dryRun(config, path) { +function dryRun(config, path, params) { path = url.parse(path, true); - return { - url: url.format({ - protocol: config.protocol, - hostname: config.hostname, - port: (config.port !== 80 && config.port !== 443 ? - config.port : undefined), - pathname: path.pathname, - query: path.query - }) - }; + if (params && params.custom) { + return { + url: url.format({ + protocol: config.protocol, + hostname: config.hostname, + port: (config.port !== 80 && config.port !== 443 ? + config.port : undefined), + pathname: path.pathname, + }), + form: params + }; + + } else { + return { + url: url.format({ + protocol: config.protocol, + hostname: config.hostname, + port: (config.port !== 80 && config.port !== 443 ? + config.port : undefined), + pathname: path.pathname, + query: params + }) + }; + } } // Normalize server config @@ -221,8 +235,8 @@ function setQuery(map, options, query) { map.options.forEach(function eachOpts(opt) { Object.keys(opt).forEach(function eachOpt(key) { var param = opt[key], - name = param.name, - value = options[name] || options[key]; + name = param.name, + value = options[name] || options[key]; if (value !== undefined && param.api) { if (param.array) { diff --git a/lib/webpagetest.js b/lib/webpagetest.js index 2f61306..2aa0d3b 100644 --- a/lib/webpagetest.js +++ b/lib/webpagetest.js @@ -13,7 +13,8 @@ var http = require("http"), specs = require("./specs"), helper = require("./helper"), server = require("./server"), - mapping = require("./mapping"); + mapping = require("./mapping"), + qs = require('querystring'); var reSpace = /\s/, reConnectivity = @@ -58,9 +59,14 @@ var filenames = { }; // GET helper function -function get(config, pathname, proxy, agent, callback, encoding) { +function get(config, pathname, query, proxy, agent, callback, encoding) { var protocol, options; + pathname = url.format({ + pathname: pathname, + query: query, + }); + if (proxy) { var proxyUrl = url.parse(proxy); var pathForProxy = config.protocol + "//"; @@ -161,6 +167,118 @@ function get(config, pathname, proxy, agent, callback, encoding) { }); } +// execute runTest using POST request +function post(config, pathname, query, proxy, agent, callback, encoding) { + var protocol, options; + + if (proxy) { + var proxyUrl = url.parse(proxy); + var pathForProxy = config.protocol + "//"; + + if (config.auth) { + pathForProxy += config.auth + "@"; + } + + pathForProxy += config.hostname + ":" + config.port + pathname; + protocol = proxyUrl.protocol === "https:" ? https : http; + + options = { + host: proxyUrl.hostname, + port: proxyUrl.port, + path: pathForProxy, + method: "POST", + headers: { + Host: config.hostname, + }, + + }; + } else { + protocol = config.protocol === "https:" ? https : http; + options = { + path: pathname, + host: config.hostname, + auth: config.auth, + port: config.port, + method: "POST", + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + }; + } + + if (encoding !== "binary") { + options.headers["X-WPT-API-KEY"] = this.config.key; + options.headers["accept-encoding"] = "gzip,deflate"; + options.headers["User-Agent"] = "WebpagetestNodeWrapper/v0.6.0"; + } + + if (agent) { + options.agent = agent; + } + + postData = qs.stringify(query) + + return protocol + .request(options, function getResponse(res) { + var data, + length, + statusCode = res.statusCode; + + if (statusCode !== 200) { + callback( + new helper.WPTAPIError(statusCode, http.STATUS_CODES[statusCode]) + ); + } else { + data = []; + length = 0; + + encoding = res.headers["content-encoding"] || encoding || "uft8"; + + res.on("data", function onData(chunk) { + data.push(chunk); + length += chunk.length; + }); + + res.on("end", function onEnd() { + var i, + len, + pos, + buffer = new Buffer.alloc(length), + type = (res.headers["content-type"] || "").split(";")[0]; + + for (i = 0, len = data.length, pos = 0; i < len; i += 1) { + data[i].copy(buffer, pos); + pos += data[i].length; + } + + if (encoding === "gzip" || encoding === "deflate") { + // compressed response (gzip,deflate) + zlib.unzip(buffer, function unzip(err, buffer) { + if (err) { + callback(err); + } else { + callback(undefined, buffer.toString(), { + type: type, + encoding: encoding, + }); + } + }); + } else { + // uncompressed response + callback(undefined, buffer, { + type: type, + encoding: encoding, + }); + } + }); + } + }) + .on("error", function onError(err) { + callback(err); + }) + .end(postData); +} + // execute callback properly normalizing optional args function callbackYield(callback, err, data, options) { if (typeof callback === "function") { @@ -186,22 +304,20 @@ function api(pathname, callback, query, options) { config = this.config; } - pathname = url.format({ - pathname: url.resolve(config.pathname, pathname), - query: query, - }); + pathname = url.resolve(config.pathname, pathname); if (options.dryRun) { // dry run: return the API url (string) only if (typeof callback === "function") { - callback.apply(callback, [undefined, helper.dryRun(config, pathname)]); + callback.apply(callback, [undefined, helper.dryRun(config, pathname, query)]); } } else { // make the real API call - get.call( + (options.custom !== undefined ? post : get).call( this, config, pathname, + query, options.proxy, options.agent, function apiCallback(err, data, info) { diff --git a/test/edge-cases-test.js b/test/edge-cases-test.js index 77ab4eb..f581f26 100644 --- a/test/edge-cases-test.js +++ b/test/edge-cases-test.js @@ -85,6 +85,18 @@ describe('Edge Cases of', function() { }); }); + it('gets a test with custom metrics then returns API url and payload with custom metrics data present', function (done) { + wpt.runTest('http://foobar.com', { + dryRun: true, + custom: '[example]\nreturn 1;' + }, function (err, data) { + if (err) return done(err); + assert.equal(data.url, wptServer + 'runtest.php'); + assert.equal(data.form.custom, '[example]\nreturn 1;'); + done(); + }); + }); + }); describe('WebPageTest localhost helper', function() { From daacc925168dbd8edf8dab70a137629041598c06 Mon Sep 17 00:00:00 2001 From: max-ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 1 Aug 2023 18:43:06 +0200 Subject: [PATCH 02/10] deduplicated get and post functions --- lib/helper.js | 12 ++-- lib/webpagetest.js | 148 ++++++++-------------------------------- test/edge-cases-test.js | 5 +- 3 files changed, 38 insertions(+), 127 deletions(-) diff --git a/lib/helper.js b/lib/helper.js index 739aeb3..5cb8e6f 100644 --- a/lib/helper.js +++ b/lib/helper.js @@ -9,7 +9,8 @@ var xml2js = require('xml2js'), url = require('url'), os = require('os'), csv = require('csv'), - entities = require('entities'); + entities = require('entities'), + qs = require('querystring'); var parser = new xml2js.Parser({ explicitArray: false, mergeAttrs: true }); @@ -155,10 +156,10 @@ function scriptToString(data) { } // Build the RESTful API url call only -function dryRun(config, path, params) { +function dryRun(config, path, form) { path = url.parse(path, true); - if (params && params.custom) { + if (config.method == "POST") { return { url: url.format({ protocol: config.protocol, @@ -167,9 +168,8 @@ function dryRun(config, path, params) { config.port : undefined), pathname: path.pathname, }), - form: params + form: qs.stringify(form) }; - } else { return { url: url.format({ @@ -178,7 +178,7 @@ function dryRun(config, path, params) { port: (config.port !== 80 && config.port !== 443 ? config.port : undefined), pathname: path.pathname, - query: params + query: path.query }) }; } diff --git a/lib/webpagetest.js b/lib/webpagetest.js index 2aa0d3b..fb4477e 100644 --- a/lib/webpagetest.js +++ b/lib/webpagetest.js @@ -58,15 +58,10 @@ var filenames = { cached: "_Cached", }; -// GET helper function -function get(config, pathname, query, proxy, agent, callback, encoding) { +// GET/POST helper function +function get(config, pathname, data, proxy, agent, callback, encoding) { var protocol, options; - pathname = url.format({ - pathname: pathname, - query: query, - }); - if (proxy) { var proxyUrl = url.parse(proxy); var pathForProxy = config.protocol + "//"; @@ -85,6 +80,8 @@ function get(config, pathname, query, proxy, agent, callback, encoding) { headers: { Host: config.hostname, }, + method: config.method + }; } else { protocol = config.protocol === "https:" ? https : http; @@ -93,117 +90,13 @@ function get(config, pathname, query, proxy, agent, callback, encoding) { host: config.hostname, auth: config.auth, port: config.port, + method: config.method, headers: {}, }; } - if (encoding !== "binary") { - options.headers["X-WPT-API-KEY"] = this.config.key; - options.headers["accept-encoding"] = "gzip,deflate"; - options.headers["User-Agent"] = "WebpagetestNodeWrapper/v0.6.0"; - } - - if (agent) { - options.agent = agent; - } - - return protocol - .get(options, function getResponse(res) { - var data, - length, - statusCode = res.statusCode; - - if (statusCode !== 200) { - callback( - new helper.WPTAPIError(statusCode, http.STATUS_CODES[statusCode]) - ); - } else { - data = []; - length = 0; - - encoding = res.headers["content-encoding"] || encoding || "uft8"; - - res.on("data", function onData(chunk) { - data.push(chunk); - length += chunk.length; - }); - - res.on("end", function onEnd() { - var i, - len, - pos, - buffer = new Buffer.alloc(length), - type = (res.headers["content-type"] || "").split(";")[0]; - - for (i = 0, len = data.length, pos = 0; i < len; i += 1) { - data[i].copy(buffer, pos); - pos += data[i].length; - } - - if (encoding === "gzip" || encoding === "deflate") { - // compressed response (gzip,deflate) - zlib.unzip(buffer, function unzip(err, buffer) { - if (err) { - callback(err); - } else { - callback(undefined, buffer.toString(), { - type: type, - encoding: encoding, - }); - } - }); - } else { - // uncompressed response - callback(undefined, buffer, { - type: type, - encoding: encoding, - }); - } - }); - } - }) - .on("error", function onError(err) { - callback(err); - }); -} - -// execute runTest using POST request -function post(config, pathname, query, proxy, agent, callback, encoding) { - var protocol, options; - - if (proxy) { - var proxyUrl = url.parse(proxy); - var pathForProxy = config.protocol + "//"; - - if (config.auth) { - pathForProxy += config.auth + "@"; - } - - pathForProxy += config.hostname + ":" + config.port + pathname; - protocol = proxyUrl.protocol === "https:" ? https : http; - - options = { - host: proxyUrl.hostname, - port: proxyUrl.port, - path: pathForProxy, - method: "POST", - headers: { - Host: config.hostname, - }, - - }; - } else { - protocol = config.protocol === "https:" ? https : http; - options = { - path: pathname, - host: config.hostname, - auth: config.auth, - port: config.port, - method: "POST", - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - }, - }; + if (options.method == "POST") { + options.headers['Content-Type'] = 'application/x-www-form-urlencoded'; } if (encoding !== "binary") { @@ -216,9 +109,7 @@ function post(config, pathname, query, proxy, agent, callback, encoding) { options.agent = agent; } - postData = qs.stringify(query) - - return protocol + var request = protocol .request(options, function getResponse(res) { var data, length, @@ -276,7 +167,13 @@ function post(config, pathname, query, proxy, agent, callback, encoding) { .on("error", function onError(err) { callback(err); }) - .end(postData); + + if (options.method == "POST") { + return request.end(qs.stringify(data)); + } else { + return request.end(); + } + } // execute callback properly normalizing optional args @@ -306,6 +203,19 @@ function api(pathname, callback, query, options) { pathname = url.resolve(config.pathname, pathname); + config.method = url.format({ + pathname: pathname, + query: query, + }).toString().length > 6 * 1024 ? "POST" : "GET"; + + if (config.method == "GET") { + pathname = url.format({ + pathname: pathname, + query: query, + }); + query = undefined; + } + if (options.dryRun) { // dry run: return the API url (string) only if (typeof callback === "function") { @@ -313,7 +223,7 @@ function api(pathname, callback, query, options) { } } else { // make the real API call - (options.custom !== undefined ? post : get).call( + get.call( this, config, pathname, diff --git a/test/edge-cases-test.js b/test/edge-cases-test.js index f581f26..cfb4c21 100644 --- a/test/edge-cases-test.js +++ b/test/edge-cases-test.js @@ -88,11 +88,12 @@ describe('Edge Cases of', function() { it('gets a test with custom metrics then returns API url and payload with custom metrics data present', function (done) { wpt.runTest('http://foobar.com', { dryRun: true, - custom: '[example]\nreturn 1;' + mobile: 1, + custom: '[example]\n\\\\' + 'X'.repeat(6 * 1024) + '\nreturn 1;' }, function (err, data) { if (err) return done(err); assert.equal(data.url, wptServer + 'runtest.php'); - assert.equal(data.form.custom, '[example]\nreturn 1;'); + assert.equal(data.form.length, 6233); done(); }); }); From b28c65b847fd9a3bc64138dbc373efbb6a68b0c7 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 1 Nov 2023 18:02:26 +0100 Subject: [PATCH 03/10] Post request support for run test --- lib/helper.js | 38 ++++++++++++++++++++++++------------ lib/webpagetest.js | 43 ++++++++++++++++++++++++++++++++--------- test/edge-cases-test.js | 13 +++++++++++++ 3 files changed, 73 insertions(+), 21 deletions(-) diff --git a/lib/helper.js b/lib/helper.js index f9a67ce..d140238 100644 --- a/lib/helper.js +++ b/lib/helper.js @@ -9,7 +9,8 @@ var xml2js = require('xml2js'), url = require('url'), os = require('os'), csv = require('csv'), - entities = require('entities'); + entities = require('entities') + qs = require('querystring'); var parser = new xml2js.Parser({explicitArray: false, mergeAttrs: true}); @@ -155,19 +156,32 @@ function scriptToString(data) { } // Build the RESTful API url call only -function dryRun(config, path) { +function dryRun(config, path, form) { path = url.parse(path, true); - return { - url: url.format({ - protocol: config.protocol, - hostname: config.hostname, - port: (config.port !== 80 && config.port !== 443 ? - config.port : undefined), - pathname: path.pathname, - query: path.query - }) - }; + if (config.method == "POST") { + return { + url: url.format({ + protocol: config.protocol, + hostname: config.hostname, + port: (config.port !== 80 && config.port !== 443 ? + config.port : undefined), + pathname: path.pathname, + }), + form: qs.stringify(form) + }; + } else { + return { + url: url.format({ + protocol: config.protocol, + hostname: config.hostname, + port: (config.port !== 80 && config.port !== 443 ? + config.port : undefined), + pathname: path.pathname, + query: path.query + }) + }; + } } // Normalize server config diff --git a/lib/webpagetest.js b/lib/webpagetest.js index 2f61306..95a5f5e 100644 --- a/lib/webpagetest.js +++ b/lib/webpagetest.js @@ -13,7 +13,8 @@ var http = require("http"), specs = require("./specs"), helper = require("./helper"), server = require("./server"), - mapping = require("./mapping"); + mapping = require("./mapping"), + qs = require('querystring'); var reSpace = /\s/, reConnectivity = @@ -57,8 +58,8 @@ var filenames = { cached: "_Cached", }; -// GET helper function -function get(config, pathname, proxy, agent, callback, encoding) { +// GET/POST helper function +function get(config, pathname, data, proxy, agent, callback, encoding) { var protocol, options; if (proxy) { @@ -79,6 +80,7 @@ function get(config, pathname, proxy, agent, callback, encoding) { headers: { Host: config.hostname, }, + method: config.method }; } else { protocol = config.protocol === "https:" ? https : http; @@ -87,10 +89,15 @@ function get(config, pathname, proxy, agent, callback, encoding) { host: config.hostname, auth: config.auth, port: config.port, + method: config.method, headers: {}, }; } + if (options.method == "POST") { + options.headers['Content-Type'] = 'application/x-www-form-urlencoded'; + } + if (encoding !== "binary") { options.headers["X-WPT-API-KEY"] = this.config.key; options.headers["accept-encoding"] = "gzip,deflate"; @@ -101,8 +108,8 @@ function get(config, pathname, proxy, agent, callback, encoding) { options.agent = agent; } - return protocol - .get(options, function getResponse(res) { + var request = protocol + .request(options, function getResponse(res) { var data, length, statusCode = res.statusCode; @@ -159,6 +166,13 @@ function get(config, pathname, proxy, agent, callback, encoding) { .on("error", function onError(err) { callback(err); }); + + if (options.method == "POST") { + return request.end(qs.stringify(data)); + } else { + return request.end(); + } + } // execute callback properly normalizing optional args @@ -186,15 +200,25 @@ function api(pathname, callback, query, options) { config = this.config; } - pathname = url.format({ - pathname: url.resolve(config.pathname, pathname), + pathname = url.resolve(config.pathname, pathname); + + config.method = url.format({ + pathname: pathname, query: query, - }); + }).toString().length > 6 * 1024 ? "POST" : "GET"; + + if (config.method == "GET") { + pathname = url.format({ + pathname: pathname, + query: query, + }); + query = undefined; + } if (options.dryRun) { // dry run: return the API url (string) only if (typeof callback === "function") { - callback.apply(callback, [undefined, helper.dryRun(config, pathname)]); + callback.apply(callback, [undefined, helper.dryRun(config, pathname, query)]); } } else { // make the real API call @@ -202,6 +226,7 @@ function api(pathname, callback, query, options) { this, config, pathname, + query, options.proxy, options.agent, function apiCallback(err, data, info) { diff --git a/test/edge-cases-test.js b/test/edge-cases-test.js index 77ab4eb..cfb4c21 100644 --- a/test/edge-cases-test.js +++ b/test/edge-cases-test.js @@ -85,6 +85,19 @@ describe('Edge Cases of', function() { }); }); + it('gets a test with custom metrics then returns API url and payload with custom metrics data present', function (done) { + wpt.runTest('http://foobar.com', { + dryRun: true, + mobile: 1, + custom: '[example]\n\\\\' + 'X'.repeat(6 * 1024) + '\nreturn 1;' + }, function (err, data) { + if (err) return done(err); + assert.equal(data.url, wptServer + 'runtest.php'); + assert.equal(data.form.length, 6233); + done(); + }); + }); + }); describe('WebPageTest localhost helper', function() { From dee4ed05623dd1cfcef6b36fca729db04450bc72 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Fri, 3 Nov 2023 13:08:27 +0100 Subject: [PATCH 04/10] early exit --- lib/helper.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/helper.js b/lib/helper.js index d140238..2e0ab53 100644 --- a/lib/helper.js +++ b/lib/helper.js @@ -170,18 +170,18 @@ function dryRun(config, path, form) { }), form: qs.stringify(form) }; - } else { - return { - url: url.format({ - protocol: config.protocol, - hostname: config.hostname, - port: (config.port !== 80 && config.port !== 443 ? - config.port : undefined), - pathname: path.pathname, - query: path.query - }) - }; } + + return { + url: url.format({ + protocol: config.protocol, + hostname: config.hostname, + port: (config.port !== 80 && config.port !== 443 ? + config.port : undefined), + pathname: path.pathname, + query: path.query + }) + }; } // Normalize server config From 0817f400cb33a2e4ee236f5eb2b935fdc4a7dcf4 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Fri, 3 Nov 2023 14:41:55 +0100 Subject: [PATCH 05/10] method in configs --- README.md | 2 ++ lib/mapping.js | 4 ++++ lib/webpagetest.js | 18 +++++++++++------- test/edge-cases-test.js | 1 + 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index e863a25..a784faa 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,7 @@ webpagetest --help - **-s, --server** _\_: the WPT server URL [https://www.webpagetest.org] - **-d, --dryrun**: just return the RESTful API URL - **-o, --out** _\_: place the output into \. Defaults to stdout +- **--http_method** _\_: the HTTP method to use (GET, POST) [GET] _The default WPT server can also be specified via environment variable `WEBPAGETEST_SERVER`_ @@ -522,6 +523,7 @@ wpt.runTest(script, (err, data) => { - **dryRun**: _Boolean_, if `true`, method does not make an actual request to the API Server but rather returns an object with `url` which contains the actual URL to make the GET request to WebPageTest API Server - **server**: _String_, if specified, overrides the WebPageTest server informed in the constructor only for that method call +- **http_method**: _String_, if specified, overrides the HTTP method in the constructor only for that method call (GET, POST) [GET] #### Test (works with `runTest` and `runTestAndWait`) diff --git a/lib/mapping.js b/lib/mapping.js index ddd18ad..0f9844d 100644 --- a/lib/mapping.js +++ b/lib/mapping.js @@ -21,6 +21,10 @@ var options = { bool: true, info: "just return the RESTful API URL", }, + http_method: { + name: "http_method", + info: "HTTP method to use for submitting the test (GET or POST) [GET]", + } }, test: { location: { diff --git a/lib/webpagetest.js b/lib/webpagetest.js index 95a5f5e..0d5f270 100644 --- a/lib/webpagetest.js +++ b/lib/webpagetest.js @@ -19,7 +19,8 @@ var http = require("http"), var reSpace = /\s/, reConnectivity = /^(?:Cable|DSL|3GSlow|3G|3GFast|4G|LTE|Edge|2G|Dial|FIOS|Native|custom)$/, - reHTMLOutput = /([^<]+)<\/h\d>/; // for H3 on cancelTest.php + reHTMLOutput = /([^<]+)<\/h\d>/, // for H3 on cancelTest.php + reHTTPmethods = /^(?:GET|POST)$/; var paths = { testStatus: "testStatus.php", @@ -169,10 +170,10 @@ function get(config, pathname, data, proxy, agent, callback, encoding) { if (options.method == "POST") { return request.end(qs.stringify(data)); - } else { - return request.end(); } + return request.end(); + } // execute callback properly normalizing optional args @@ -202,10 +203,12 @@ function api(pathname, callback, query, options) { pathname = url.resolve(config.pathname, pathname); - config.method = url.format({ - pathname: pathname, - query: query, - }).toString().length > 6 * 1024 ? "POST" : "GET"; + if (reHTTPmethods.test(options.http_method)) { + config.method = options.http_method; + } else { + config.method = WebPageTest.defaultHTTPMethod; + } + if (config.method == "GET") { pathname = url.format({ @@ -960,6 +963,7 @@ WebPageTest.filenames = filenames; WebPageTest.defaultServer = "https://www.webpagetest.org"; WebPageTest.defaultListenPort = 7791; WebPageTest.defaultWaitResultsPort = 8000; +WebPageTest.defaultHTTPMethod = "GET"; // Version Object.defineProperty(WebPageTest, "version", { diff --git a/test/edge-cases-test.js b/test/edge-cases-test.js index cfb4c21..454cdec 100644 --- a/test/edge-cases-test.js +++ b/test/edge-cases-test.js @@ -89,6 +89,7 @@ describe('Edge Cases of', function() { wpt.runTest('http://foobar.com', { dryRun: true, mobile: 1, + http_method: 'POST', custom: '[example]\n\\\\' + 'X'.repeat(6 * 1024) + '\nreturn 1;' }, function (err, data) { if (err) return done(err); From 49444f2ffd3ad0d6364cdbd23fc06d866c3c1ec7 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Fri, 3 Nov 2023 15:26:49 +0100 Subject: [PATCH 06/10] fixed in command line --- bin/webpagetest | 2 ++ test/command-line-test.js | 11 +++++++++++ test/edge-cases-test.js | 6 +++--- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/bin/webpagetest b/bin/webpagetest index 7b32466..03751df 100755 --- a/bin/webpagetest +++ b/bin/webpagetest @@ -269,6 +269,7 @@ function cleanupProgram() { delete program.server; delete program.dryrun; delete program.out; + delete program.http_method; } function execCommand() { @@ -277,6 +278,7 @@ function execCommand() { // options if (options) { options.dryRun = program.dryrun; + options.http_method = program.http_method; args.push(options); } diff --git a/test/command-line-test.js b/test/command-line-test.js index 578796f..836cf84 100644 --- a/test/command-line-test.js +++ b/test/command-line-test.js @@ -352,4 +352,15 @@ describe('WebPageTest Command Line', function() { }); }); + it('gets a test with long custom metrics script then returns API url and payload with custom metrics data', function (done) { + let script = '"[example]\n\\\\' + 'X'.repeat(10000) + '\nreturn 1;"' + exec(mock('test http://foobar.com --http_method POST --custom ' + script), function (err, data) { + if (err) return done(err); + data = JSON.parse(data); + assert.equal(data.url, wptServer + 'runtest.php'); + assert.equal(data.form.length, 10089); + done(); + }); + }); + }); diff --git a/test/edge-cases-test.js b/test/edge-cases-test.js index 454cdec..fc97132 100644 --- a/test/edge-cases-test.js +++ b/test/edge-cases-test.js @@ -85,16 +85,16 @@ describe('Edge Cases of', function() { }); }); - it('gets a test with custom metrics then returns API url and payload with custom metrics data present', function (done) { + it('gets a test with long custom metrics script then returns API url and payload with custom metrics data', function (done) { wpt.runTest('http://foobar.com', { dryRun: true, mobile: 1, http_method: 'POST', - custom: '[example]\n\\\\' + 'X'.repeat(6 * 1024) + '\nreturn 1;' + custom: '[example]\n\\\\' + 'X'.repeat(10000) + '\nreturn 1;' }, function (err, data) { if (err) return done(err); assert.equal(data.url, wptServer + 'runtest.php'); - assert.equal(data.form.length, 6233); + assert.equal(data.form.length, 10089); done(); }); }); From a959c18359acfc28dd22db60ab8087781efb0c3f Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Fri, 3 Nov 2023 15:57:55 +0100 Subject: [PATCH 07/10] missing param --- lib/mapping.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/mapping.js b/lib/mapping.js index 0f9844d..66313ed 100644 --- a/lib/mapping.js +++ b/lib/mapping.js @@ -23,6 +23,7 @@ var options = { }, http_method: { name: "http_method", + param: "string", info: "HTTP method to use for submitting the test (GET or POST) [GET]", } }, From f675b56322c2c7214f5926ebf642a1e028f750a2 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Fri, 3 Nov 2023 16:07:20 +0100 Subject: [PATCH 08/10] shorter payload --- test/command-line-test.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/command-line-test.js b/test/command-line-test.js index 836cf84..37795eb 100644 --- a/test/command-line-test.js +++ b/test/command-line-test.js @@ -353,12 +353,13 @@ describe('WebPageTest Command Line', function() { }); it('gets a test with long custom metrics script then returns API url and payload with custom metrics data', function (done) { - let script = '"[example]\n\\\\' + 'X'.repeat(10000) + '\nreturn 1;"' + let script = '"[example]\n\\\\' + 'X'.repeat(6000) + '\nreturn 1;"' + exec(mock('test http://foobar.com --http_method POST --custom ' + script), function (err, data) { if (err) return done(err); data = JSON.parse(data); assert.equal(data.url, wptServer + 'runtest.php'); - assert.equal(data.form.length, 10089); + assert.equal(data.form.length, 6077); done(); }); }); From 39f95a9b4d2cf9bd02e47fce2a31733d4f5992a0 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 14 Nov 2023 19:21:41 +0100 Subject: [PATCH 09/10] http_method option --- test/fixtures/command-line/help.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/fixtures/command-line/help.txt b/test/fixtures/command-line/help.txt index 185ea83..000c1fd 100644 --- a/test/fixtures/command-line/help.txt +++ b/test/fixtures/command-line/help.txt @@ -5,6 +5,8 @@ Options: -s, --server the WPT server URL [https://www.webpagetest.org] -d, --dryrun just return the RESTful API URL + --http_method HTTP method to use for submitting the test + (GET or POST) [GET] -o, --out place the output into . Defaults to stdout -h, --help display help for command From a521b5e37116f20e71dab1f0ee8ae6514a9f3d36 Mon Sep 17 00:00:00 2001 From: Max Ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Wed, 8 May 2024 02:46:29 +0200 Subject: [PATCH 10/10] added wappalyzerpr to test runs --- README.md | 4 +++- lib/mapping.js | 15 +++++++++++++++ test/command-line-test.js | 3 ++- test/edge-cases-test.js | 10 ++++++++++ test/fixtures/command-line/help-test.txt | 6 ++++-- test/fixtures/command-line/help-testAndWait.txt | 6 ++++-- 6 files changed, 38 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7fbe1f6..f2a30d1 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,7 @@ _The default WPT server can also be specified via environment variable `WEBPAGET - **-b, --block** _\_: space-delimited list of urls to block (substring match) - **-Z, --spof** _\_: space-delimited list of domains to simulate failure by re-routing to blackhole.webpagetest.org to silently drop all requests - **-c, --custom** _\_: execute arbitrary javascript at the end of a test to collect custom metrics +- **-w, --wappalyzerpr** _\_: set the wappalyzer fork PR number to use for technology detection - **-a, --authtype** _\_: type of authentication: 0 = Basic, 1 = SNS [0] - **-n, --notify** _\_: e-mail address to notify with the test results - **-B, --pingback** _\_: URL to ping when the test is complete (the test ID will be passed as an "id" parameter) @@ -317,7 +318,7 @@ webpagetest status 121025_PT_N8K -k YOURAPIKEY #### 4. Get test results ```bash -webpagetest results 121025_PT_N8K -k YOURAPIKEY +webpagetest results 121025_PT_N8K -k YOURAPIKEY ``` ```javascript @@ -519,6 +520,7 @@ wpt.runTest(script, (err, data) => { - **block**: _String_, space-delimited list of urls to block (substring match) - **spof**: _String_, space-delimited list of domains to simulate failure by re-routing to blackhole.webpagetest.org to silently drop all requests - **customMetrics**: _String_, execute arbitrary JavaScript at the end of a test to collect custom metrics +- **wappalyzerpr** _Integer_, set the wappalyzer fork PR number to use for technology detection - **authenticationType**: _Number_, type of authentication: 0 = Basic, 1 = SNS [0] - **notifyEmail**: _String_, e-mail address to notify with the test results - **pingback**: _String_, URL to ping when the test is complete (the test ID will be passed as an "id" parameter) diff --git a/lib/mapping.js b/lib/mapping.js index 5908c77..868b854 100644 --- a/lib/mapping.js +++ b/lib/mapping.js @@ -312,6 +312,14 @@ var options = { info: "execute arbitrary JavaScript at the end of a test to collect custom metrics", }, + wappalyzerpr: { + name: "wappalyzerpr", + key: "wpr", + api: "wappalyzerPR", + param: "number", + info: "PR number of the Wappalyzer fork to use in technology detection", + }, + // API only settings authtype: { name: "authenticationType", @@ -841,6 +849,13 @@ var options = { param: "script", info: "execute arbitrary JavaScript at the end of a test to collect custom metrics", }, + wappalyzerpr: { + name: "wappalyzerpr", + key: "wpr", + api: "wappalyzerPR", + param: "number", + info: "PR number of the Wappalyzer fork to use in technology detection", + }, // API only settings authtype: { diff --git a/test/command-line-test.js b/test/command-line-test.js index 37795eb..f6c8abc 100644 --- a/test/command-line-test.js +++ b/test/command-line-test.js @@ -102,11 +102,12 @@ describe('WebPageTest Command Line', function() { '--first ' + '--timeline ' + '--netlog ' + + '--wappalyzerpr 1 ' + '--full' ), function(err, data) { if (err) return done(err); data = JSON.parse(data); - assert.equal(data.url, wptServer + 'runtest.php?url=http%3A%2F%2Ftwitter.com%2Fmarcelduran&location=Local_Firefox_Chrome%3AChrome&runs=3&fvonly=1&label=test%20123&timeline=1&netlog=1&pngss=1&f=json'); + assert.equal(data.url, wptServer + 'runtest.php?url=http%3A%2F%2Ftwitter.com%2Fmarcelduran&location=Local_Firefox_Chrome%3AChrome&runs=3&fvonly=1&label=test%20123&timeline=1&netlog=1&wappalyzerPR=1&pngss=1&f=json'); done(); }); }); diff --git a/test/edge-cases-test.js b/test/edge-cases-test.js index fc97132..653b31a 100644 --- a/test/edge-cases-test.js +++ b/test/edge-cases-test.js @@ -99,6 +99,16 @@ describe('Edge Cases of', function() { }); }); + it('gets a test with custom wappalyzer rules then returns API url', function (done) { + wpt.runTest('http://foobar.com', { + dryRun: true, + wappalyzerpr: 1, + }, function (err, data) { + if (err) return done(err); + assert.equal(data.url, wptServer + 'runtest.php?url=http%3A%2F%2Ffoobar.com&wappalyzerPR=1&f=json'); + done(); + }); + }); }); describe('WebPageTest localhost helper', function() { diff --git a/test/fixtures/command-line/help-test.txt b/test/fixtures/command-line/help-test.txt index ed98f04..15d6e64 100644 --- a/test/fixtures/command-line/help-test.txt +++ b/test/fixtures/command-line/help-test.txt @@ -60,7 +60,7 @@ Options: switches (Chrome only) --lighthouse perform lighthouse test (Chrome only, Linux agent only) - -thc, --throttleCPU custom cpu throttling + -thc, --throttleCPU custom cpu throttling -g, --login username for authenticating tests (http authentication) -w, --password password for authenticating tests (http @@ -78,6 +78,8 @@ Options: drop all requests -c, --custom