From 27d3353655ccfee2a343795897561071ef527bd8 Mon Sep 17 00:00:00 2001 From: max-ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:14:11 +0200 Subject: [PATCH 1/2] conditional post requests + test --- lib/helper.js | 78 ++++++++++++++---------- lib/webpagetest.js | 132 +++++++++++++++++++++++++++++++++++++--- test/edge-cases-test.js | 12 ++++ 3 files changed, 182 insertions(+), 40 deletions(-) diff --git a/lib/helper.js b/lib/helper.js index f9a67ce..739aeb3 100644 --- a/lib/helper.js +++ b/lib/helper.js @@ -5,24 +5,24 @@ * Released under the MIT License */ -var xml2js = require('xml2js'), - url = require('url'), - os = require('os'), - csv = require('csv'), - entities = require('entities'); +var xml2js = require('xml2js'), + url = require('url'), + os = require('os'), + csv = require('csv'), + entities = require('entities'); -var parser = new xml2js.Parser({explicitArray: false, mergeAttrs: true}); +var parser = new xml2js.Parser({ explicitArray: false, mergeAttrs: true }); var reNumber = /^[\.\+\-]?[\d\.]+$/, - reInvalidDec = /(?:\.\d*){2,}/, - reDec = /\./, - reLineBreak = /[\n\r]+/g, - reLastBreak = /\n$/, - reProtocol = /^https?:\/\//i, - reIp = /\d+\.\d+\.\d+\.\d+/, // 127.0.0.1 + reInvalidDec = /(?:\.\d*){2,}/, + reDec = /\./, + reLineBreak = /[\n\r]+/g, + reLastBreak = /\n$/, + reProtocol = /^https?:\/\//i, + reIp = /\d+\.\d+\.\d+\.\d+/, // 127.0.0.1 - TAB = '\t', - NEWLINE = '\n'; + TAB = '\t', + NEWLINE = '\n'; function parseNumber(s) { if (typeof s !== 'string' || !reNumber.test(s) || reInvalidDec.test(s)) { @@ -34,7 +34,7 @@ function parseNumber(s) { function normalizeObj(root) { if (typeof root === 'object') { - Object.keys(root).forEach(function(key) { + Object.keys(root).forEach(function (key) { var value = root[key]; if (typeof value === 'string') { if (value.length === 0 || value === '\n') { @@ -62,8 +62,8 @@ function xmlToObj(xml, callback) { function svToObj(delimiter, headers, sv) { var data, - start = 0, - obj = {}; + start = 0, + obj = {}; delimiter = delimiter || ','; @@ -101,11 +101,11 @@ function svToObj(delimiter, headers, sv) { } function csvParser(data, callback) { - csv.parse(data.toString(), { columns: true }, function(err, data) { + csv.parse(data.toString(), { columns: true }, function (err, data) { if (err) { callback.bind(this, err); } - csv.transform(data, function(row) { + csv.transform(data, function (row) { var key, value; for (key in row) { value = row[key].replace(/|<\/b>/g, ''); @@ -155,19 +155,33 @@ function scriptToString(data) { } // Build the RESTful API url call only -function dryRun(config, path) { +function dryRun(config, path, params) { path = url.parse(path, true); - return { - url: url.format({ - protocol: config.protocol, - hostname: config.hostname, - port: (config.port !== 80 && config.port !== 443 ? - config.port : undefined), - pathname: path.pathname, - query: path.query - }) - }; + if (params && params.custom) { + return { + url: url.format({ + protocol: config.protocol, + hostname: config.hostname, + port: (config.port !== 80 && config.port !== 443 ? + config.port : undefined), + pathname: path.pathname, + }), + form: params + }; + + } else { + return { + url: url.format({ + protocol: config.protocol, + hostname: config.hostname, + port: (config.port !== 80 && config.port !== 443 ? + config.port : undefined), + pathname: path.pathname, + query: params + }) + }; + } } // Normalize server config @@ -221,8 +235,8 @@ function setQuery(map, options, query) { map.options.forEach(function eachOpts(opt) { Object.keys(opt).forEach(function eachOpt(key) { var param = opt[key], - name = param.name, - value = options[name] || options[key]; + name = param.name, + value = options[name] || options[key]; if (value !== undefined && param.api) { if (param.array) { diff --git a/lib/webpagetest.js b/lib/webpagetest.js index 2f61306..2aa0d3b 100644 --- a/lib/webpagetest.js +++ b/lib/webpagetest.js @@ -13,7 +13,8 @@ var http = require("http"), specs = require("./specs"), helper = require("./helper"), server = require("./server"), - mapping = require("./mapping"); + mapping = require("./mapping"), + qs = require('querystring'); var reSpace = /\s/, reConnectivity = @@ -58,9 +59,14 @@ var filenames = { }; // GET helper function -function get(config, pathname, proxy, agent, callback, encoding) { +function get(config, pathname, query, proxy, agent, callback, encoding) { var protocol, options; + pathname = url.format({ + pathname: pathname, + query: query, + }); + if (proxy) { var proxyUrl = url.parse(proxy); var pathForProxy = config.protocol + "//"; @@ -161,6 +167,118 @@ function get(config, pathname, proxy, agent, callback, encoding) { }); } +// execute runTest using POST request +function post(config, pathname, query, proxy, agent, callback, encoding) { + var protocol, options; + + if (proxy) { + var proxyUrl = url.parse(proxy); + var pathForProxy = config.protocol + "//"; + + if (config.auth) { + pathForProxy += config.auth + "@"; + } + + pathForProxy += config.hostname + ":" + config.port + pathname; + protocol = proxyUrl.protocol === "https:" ? https : http; + + options = { + host: proxyUrl.hostname, + port: proxyUrl.port, + path: pathForProxy, + method: "POST", + headers: { + Host: config.hostname, + }, + + }; + } else { + protocol = config.protocol === "https:" ? https : http; + options = { + path: pathname, + host: config.hostname, + auth: config.auth, + port: config.port, + method: "POST", + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + }; + } + + if (encoding !== "binary") { + options.headers["X-WPT-API-KEY"] = this.config.key; + options.headers["accept-encoding"] = "gzip,deflate"; + options.headers["User-Agent"] = "WebpagetestNodeWrapper/v0.6.0"; + } + + if (agent) { + options.agent = agent; + } + + postData = qs.stringify(query) + + return protocol + .request(options, function getResponse(res) { + var data, + length, + statusCode = res.statusCode; + + if (statusCode !== 200) { + callback( + new helper.WPTAPIError(statusCode, http.STATUS_CODES[statusCode]) + ); + } else { + data = []; + length = 0; + + encoding = res.headers["content-encoding"] || encoding || "uft8"; + + res.on("data", function onData(chunk) { + data.push(chunk); + length += chunk.length; + }); + + res.on("end", function onEnd() { + var i, + len, + pos, + buffer = new Buffer.alloc(length), + type = (res.headers["content-type"] || "").split(";")[0]; + + for (i = 0, len = data.length, pos = 0; i < len; i += 1) { + data[i].copy(buffer, pos); + pos += data[i].length; + } + + if (encoding === "gzip" || encoding === "deflate") { + // compressed response (gzip,deflate) + zlib.unzip(buffer, function unzip(err, buffer) { + if (err) { + callback(err); + } else { + callback(undefined, buffer.toString(), { + type: type, + encoding: encoding, + }); + } + }); + } else { + // uncompressed response + callback(undefined, buffer, { + type: type, + encoding: encoding, + }); + } + }); + } + }) + .on("error", function onError(err) { + callback(err); + }) + .end(postData); +} + // execute callback properly normalizing optional args function callbackYield(callback, err, data, options) { if (typeof callback === "function") { @@ -186,22 +304,20 @@ function api(pathname, callback, query, options) { config = this.config; } - pathname = url.format({ - pathname: url.resolve(config.pathname, pathname), - query: query, - }); + pathname = url.resolve(config.pathname, pathname); if (options.dryRun) { // dry run: return the API url (string) only if (typeof callback === "function") { - callback.apply(callback, [undefined, helper.dryRun(config, pathname)]); + callback.apply(callback, [undefined, helper.dryRun(config, pathname, query)]); } } else { // make the real API call - get.call( + (options.custom !== undefined ? post : get).call( this, config, pathname, + query, options.proxy, options.agent, function apiCallback(err, data, info) { diff --git a/test/edge-cases-test.js b/test/edge-cases-test.js index 77ab4eb..f581f26 100644 --- a/test/edge-cases-test.js +++ b/test/edge-cases-test.js @@ -85,6 +85,18 @@ describe('Edge Cases of', function() { }); }); + it('gets a test with custom metrics then returns API url and payload with custom metrics data present', function (done) { + wpt.runTest('http://foobar.com', { + dryRun: true, + custom: '[example]\nreturn 1;' + }, function (err, data) { + if (err) return done(err); + assert.equal(data.url, wptServer + 'runtest.php'); + assert.equal(data.form.custom, '[example]\nreturn 1;'); + done(); + }); + }); + }); describe('WebPageTest localhost helper', function() { From daacc925168dbd8edf8dab70a137629041598c06 Mon Sep 17 00:00:00 2001 From: max-ostapenko <1611259+max-ostapenko@users.noreply.github.com> Date: Tue, 1 Aug 2023 18:43:06 +0200 Subject: [PATCH 2/2] deduplicated get and post functions --- lib/helper.js | 12 ++-- lib/webpagetest.js | 148 ++++++++-------------------------------- test/edge-cases-test.js | 5 +- 3 files changed, 38 insertions(+), 127 deletions(-) diff --git a/lib/helper.js b/lib/helper.js index 739aeb3..5cb8e6f 100644 --- a/lib/helper.js +++ b/lib/helper.js @@ -9,7 +9,8 @@ var xml2js = require('xml2js'), url = require('url'), os = require('os'), csv = require('csv'), - entities = require('entities'); + entities = require('entities'), + qs = require('querystring'); var parser = new xml2js.Parser({ explicitArray: false, mergeAttrs: true }); @@ -155,10 +156,10 @@ function scriptToString(data) { } // Build the RESTful API url call only -function dryRun(config, path, params) { +function dryRun(config, path, form) { path = url.parse(path, true); - if (params && params.custom) { + if (config.method == "POST") { return { url: url.format({ protocol: config.protocol, @@ -167,9 +168,8 @@ function dryRun(config, path, params) { config.port : undefined), pathname: path.pathname, }), - form: params + form: qs.stringify(form) }; - } else { return { url: url.format({ @@ -178,7 +178,7 @@ function dryRun(config, path, params) { port: (config.port !== 80 && config.port !== 443 ? config.port : undefined), pathname: path.pathname, - query: params + query: path.query }) }; } diff --git a/lib/webpagetest.js b/lib/webpagetest.js index 2aa0d3b..fb4477e 100644 --- a/lib/webpagetest.js +++ b/lib/webpagetest.js @@ -58,15 +58,10 @@ var filenames = { cached: "_Cached", }; -// GET helper function -function get(config, pathname, query, proxy, agent, callback, encoding) { +// GET/POST helper function +function get(config, pathname, data, proxy, agent, callback, encoding) { var protocol, options; - pathname = url.format({ - pathname: pathname, - query: query, - }); - if (proxy) { var proxyUrl = url.parse(proxy); var pathForProxy = config.protocol + "//"; @@ -85,6 +80,8 @@ function get(config, pathname, query, proxy, agent, callback, encoding) { headers: { Host: config.hostname, }, + method: config.method + }; } else { protocol = config.protocol === "https:" ? https : http; @@ -93,117 +90,13 @@ function get(config, pathname, query, proxy, agent, callback, encoding) { host: config.hostname, auth: config.auth, port: config.port, + method: config.method, headers: {}, }; } - if (encoding !== "binary") { - options.headers["X-WPT-API-KEY"] = this.config.key; - options.headers["accept-encoding"] = "gzip,deflate"; - options.headers["User-Agent"] = "WebpagetestNodeWrapper/v0.6.0"; - } - - if (agent) { - options.agent = agent; - } - - return protocol - .get(options, function getResponse(res) { - var data, - length, - statusCode = res.statusCode; - - if (statusCode !== 200) { - callback( - new helper.WPTAPIError(statusCode, http.STATUS_CODES[statusCode]) - ); - } else { - data = []; - length = 0; - - encoding = res.headers["content-encoding"] || encoding || "uft8"; - - res.on("data", function onData(chunk) { - data.push(chunk); - length += chunk.length; - }); - - res.on("end", function onEnd() { - var i, - len, - pos, - buffer = new Buffer.alloc(length), - type = (res.headers["content-type"] || "").split(";")[0]; - - for (i = 0, len = data.length, pos = 0; i < len; i += 1) { - data[i].copy(buffer, pos); - pos += data[i].length; - } - - if (encoding === "gzip" || encoding === "deflate") { - // compressed response (gzip,deflate) - zlib.unzip(buffer, function unzip(err, buffer) { - if (err) { - callback(err); - } else { - callback(undefined, buffer.toString(), { - type: type, - encoding: encoding, - }); - } - }); - } else { - // uncompressed response - callback(undefined, buffer, { - type: type, - encoding: encoding, - }); - } - }); - } - }) - .on("error", function onError(err) { - callback(err); - }); -} - -// execute runTest using POST request -function post(config, pathname, query, proxy, agent, callback, encoding) { - var protocol, options; - - if (proxy) { - var proxyUrl = url.parse(proxy); - var pathForProxy = config.protocol + "//"; - - if (config.auth) { - pathForProxy += config.auth + "@"; - } - - pathForProxy += config.hostname + ":" + config.port + pathname; - protocol = proxyUrl.protocol === "https:" ? https : http; - - options = { - host: proxyUrl.hostname, - port: proxyUrl.port, - path: pathForProxy, - method: "POST", - headers: { - Host: config.hostname, - }, - - }; - } else { - protocol = config.protocol === "https:" ? https : http; - options = { - path: pathname, - host: config.hostname, - auth: config.auth, - port: config.port, - method: "POST", - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - }, - }; + if (options.method == "POST") { + options.headers['Content-Type'] = 'application/x-www-form-urlencoded'; } if (encoding !== "binary") { @@ -216,9 +109,7 @@ function post(config, pathname, query, proxy, agent, callback, encoding) { options.agent = agent; } - postData = qs.stringify(query) - - return protocol + var request = protocol .request(options, function getResponse(res) { var data, length, @@ -276,7 +167,13 @@ function post(config, pathname, query, proxy, agent, callback, encoding) { .on("error", function onError(err) { callback(err); }) - .end(postData); + + if (options.method == "POST") { + return request.end(qs.stringify(data)); + } else { + return request.end(); + } + } // execute callback properly normalizing optional args @@ -306,6 +203,19 @@ function api(pathname, callback, query, options) { pathname = url.resolve(config.pathname, pathname); + config.method = url.format({ + pathname: pathname, + query: query, + }).toString().length > 6 * 1024 ? "POST" : "GET"; + + if (config.method == "GET") { + pathname = url.format({ + pathname: pathname, + query: query, + }); + query = undefined; + } + if (options.dryRun) { // dry run: return the API url (string) only if (typeof callback === "function") { @@ -313,7 +223,7 @@ function api(pathname, callback, query, options) { } } else { // make the real API call - (options.custom !== undefined ? post : get).call( + get.call( this, config, pathname, diff --git a/test/edge-cases-test.js b/test/edge-cases-test.js index f581f26..cfb4c21 100644 --- a/test/edge-cases-test.js +++ b/test/edge-cases-test.js @@ -88,11 +88,12 @@ describe('Edge Cases of', function() { it('gets a test with custom metrics then returns API url and payload with custom metrics data present', function (done) { wpt.runTest('http://foobar.com', { dryRun: true, - custom: '[example]\nreturn 1;' + mobile: 1, + custom: '[example]\n\\\\' + 'X'.repeat(6 * 1024) + '\nreturn 1;' }, function (err, data) { if (err) return done(err); assert.equal(data.url, wptServer + 'runtest.php'); - assert.equal(data.form.custom, '[example]\nreturn 1;'); + assert.equal(data.form.length, 6233); done(); }); });