diff --git a/README.md b/README.md index b5bef8c..f2a30d1 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,7 @@ webpagetest --help - **-s, --server** _\_: the WPT server URL [https://www.webpagetest.org] - **-d, --dryrun**: just return the RESTful API URL - **-o, --out** _\_: place the output into \. Defaults to stdout +- **--http_method** _\_: the HTTP method to use (GET, POST) [GET] _The default WPT server can also be specified via environment variable `WEBPAGETEST_SERVER`_ @@ -139,6 +140,7 @@ _The default WPT server can also be specified via environment variable `WEBPAGET - **-b, --block** _\_: space-delimited list of urls to block (substring match) - **-Z, --spof** _\_: space-delimited list of domains to simulate failure by re-routing to blackhole.webpagetest.org to silently drop all requests - **-c, --custom** _\_: execute arbitrary javascript at the end of a test to collect custom metrics +- **-w, --wappalyzerpr** _\_: set the wappalyzer fork PR number to use for technology detection - **-a, --authtype** _\_: type of authentication: 0 = Basic, 1 = SNS [0] - **-n, --notify** _\_: e-mail address to notify with the test results - **-B, --pingback** _\_: URL to ping when the test is complete (the test ID will be passed as an "id" parameter) @@ -316,7 +318,7 @@ webpagetest status 121025_PT_N8K -k YOURAPIKEY #### 4. Get test results ```bash -webpagetest results 121025_PT_N8K -k YOURAPIKEY +webpagetest results 121025_PT_N8K -k YOURAPIKEY ``` ```javascript @@ -480,6 +482,7 @@ wpt.runTest(script, (err, data) => { - **dryRun**: _Boolean_, if `true`, method does not make an actual request to the API Server but rather returns an object with `url` which contains the actual URL to make the GET request to WebPageTest API Server - **server**: _String_, if specified, overrides the WebPageTest server informed in the constructor only for that method call +- **http_method**: _String_, if specified, overrides the HTTP method in the constructor only for that method call (GET, POST) [GET] #### Test (works with `runTest` and `runTestAndWait`) @@ -517,6 +520,7 @@ wpt.runTest(script, (err, data) => { - **block**: _String_, space-delimited list of urls to block (substring match) - **spof**: _String_, space-delimited list of domains to simulate failure by re-routing to blackhole.webpagetest.org to silently drop all requests - **customMetrics**: _String_, execute arbitrary JavaScript at the end of a test to collect custom metrics +- **wappalyzerpr** _Integer_, set the wappalyzer fork PR number to use for technology detection - **authenticationType**: _Number_, type of authentication: 0 = Basic, 1 = SNS [0] - **notifyEmail**: _String_, e-mail address to notify with the test results - **pingback**: _String_, URL to ping when the test is complete (the test ID will be passed as an "id" parameter) diff --git a/bin/webpagetest b/bin/webpagetest index 004078a..62ceec0 100755 --- a/bin/webpagetest +++ b/bin/webpagetest @@ -269,6 +269,7 @@ function cleanupProgram() { delete program.server; delete program.dryrun; delete program.out; + delete program.http_method; } function execCommand() { @@ -277,6 +278,7 @@ function execCommand() { // options if (options) { options.dryRun = program.dryrun; + options.http_method = program.http_method; args.push(options); } diff --git a/lib/helper.js b/lib/helper.js index f9a67ce..2e0ab53 100644 --- a/lib/helper.js +++ b/lib/helper.js @@ -9,7 +9,8 @@ var xml2js = require('xml2js'), url = require('url'), os = require('os'), csv = require('csv'), - entities = require('entities'); + entities = require('entities') + qs = require('querystring'); var parser = new xml2js.Parser({explicitArray: false, mergeAttrs: true}); @@ -155,9 +156,22 @@ function scriptToString(data) { } // Build the RESTful API url call only -function dryRun(config, path) { +function dryRun(config, path, form) { path = url.parse(path, true); + if (config.method == "POST") { + return { + url: url.format({ + protocol: config.protocol, + hostname: config.hostname, + port: (config.port !== 80 && config.port !== 443 ? + config.port : undefined), + pathname: path.pathname, + }), + form: qs.stringify(form) + }; + } + return { url: url.format({ protocol: config.protocol, diff --git a/lib/mapping.js b/lib/mapping.js index 52b6f46..868b854 100644 --- a/lib/mapping.js +++ b/lib/mapping.js @@ -21,6 +21,11 @@ var options = { bool: true, info: "just return the RESTful API URL", }, + http_method: { + name: "http_method", + param: "string", + info: "HTTP method to use for submitting the test (GET or POST) [GET]", + } }, test: { location: { @@ -307,6 +312,14 @@ var options = { info: "execute arbitrary JavaScript at the end of a test to collect custom metrics", }, + wappalyzerpr: { + name: "wappalyzerpr", + key: "wpr", + api: "wappalyzerPR", + param: "number", + info: "PR number of the Wappalyzer fork to use in technology detection", + }, + // API only settings authtype: { name: "authenticationType", @@ -836,6 +849,13 @@ var options = { param: "script", info: "execute arbitrary JavaScript at the end of a test to collect custom metrics", }, + wappalyzerpr: { + name: "wappalyzerpr", + key: "wpr", + api: "wappalyzerPR", + param: "number", + info: "PR number of the Wappalyzer fork to use in technology detection", + }, // API only settings authtype: { diff --git a/lib/webpagetest.js b/lib/webpagetest.js index c593336..4705a71 100644 --- a/lib/webpagetest.js +++ b/lib/webpagetest.js @@ -13,12 +13,14 @@ var http = require("http"), specs = require("./specs"), helper = require("./helper"), server = require("./server"), - mapping = require("./mapping"); + mapping = require("./mapping"), + qs = require('querystring'); var reSpace = /\s/, reConnectivity = /^(?:Cable|DSL|3GSlow|3G|3GFast|4G|LTE|Edge|2G|Dial|FIOS|Native|custom)$/, - reHTMLOutput = /([^<]+)<\/h\d>/; // for H3 on cancelTest.php + reHTMLOutput = /([^<]+)<\/h\d>/, // for H3 on cancelTest.php + reHTTPmethods = /^(?:GET|POST)$/; var paths = { testStatus: "testStatus.php", @@ -57,8 +59,8 @@ var filenames = { cached: "_Cached", }; -// GET helper function -function get(config, pathname, proxy, agent, callback, encoding) { +// GET/POST helper function +function get(config, pathname, data, proxy, agent, callback, encoding) { var protocol, options; if (proxy) { @@ -79,6 +81,7 @@ function get(config, pathname, proxy, agent, callback, encoding) { headers: { Host: config.hostname, }, + method: config.method }; } else { protocol = config.protocol === "https:" ? https : http; @@ -87,10 +90,15 @@ function get(config, pathname, proxy, agent, callback, encoding) { host: config.hostname, auth: config.auth, port: config.port, + method: config.method, headers: {}, }; } + if (options.method == "POST") { + options.headers['Content-Type'] = 'application/x-www-form-urlencoded'; + } + // api key always required options.headers["X-WPT-API-KEY"] = this.config.key; options.headers["accept-encoding"] = "gzip,deflate"; @@ -100,8 +108,8 @@ function get(config, pathname, proxy, agent, callback, encoding) { options.agent = agent; } - return protocol - .get(options, function getResponse(res) { + var request = protocol + .request(options, function getResponse(res) { var data, length, statusCode = res.statusCode; @@ -158,6 +166,13 @@ function get(config, pathname, proxy, agent, callback, encoding) { .on("error", function onError(err) { callback(err); }); + + if (options.method == "POST") { + return request.end(qs.stringify(data)); + } + + return request.end(); + } // execute callback properly normalizing optional args @@ -185,15 +200,27 @@ function api(pathname, callback, query, options) { config = this.config; } - pathname = url.format({ - pathname: url.resolve(config.pathname, pathname), - query: query, - }); + pathname = url.resolve(config.pathname, pathname); + + if (reHTTPmethods.test(options.http_method)) { + config.method = options.http_method; + } else { + config.method = WebPageTest.defaultHTTPMethod; + } + + + if (config.method == "GET") { + pathname = url.format({ + pathname: pathname, + query: query, + }); + query = undefined; + } if (options.dryRun) { // dry run: return the API url (string) only if (typeof callback === "function") { - callback.apply(callback, [undefined, helper.dryRun(config, pathname)]); + callback.apply(callback, [undefined, helper.dryRun(config, pathname, query)]); } } else { // make the real API call @@ -201,6 +228,7 @@ function api(pathname, callback, query, options) { this, config, pathname, + query, options.proxy, options.agent, function apiCallback(err, data, info) { @@ -931,6 +959,7 @@ WebPageTest.filenames = filenames; WebPageTest.defaultServer = "https://www.webpagetest.org"; WebPageTest.defaultListenPort = 7791; WebPageTest.defaultWaitResultsPort = 8000; +WebPageTest.defaultHTTPMethod = "GET"; // Version Object.defineProperty(WebPageTest, "version", { diff --git a/test/command-line-test.js b/test/command-line-test.js index 578796f..f6c8abc 100644 --- a/test/command-line-test.js +++ b/test/command-line-test.js @@ -102,11 +102,12 @@ describe('WebPageTest Command Line', function() { '--first ' + '--timeline ' + '--netlog ' + + '--wappalyzerpr 1 ' + '--full' ), function(err, data) { if (err) return done(err); data = JSON.parse(data); - assert.equal(data.url, wptServer + 'runtest.php?url=http%3A%2F%2Ftwitter.com%2Fmarcelduran&location=Local_Firefox_Chrome%3AChrome&runs=3&fvonly=1&label=test%20123&timeline=1&netlog=1&pngss=1&f=json'); + assert.equal(data.url, wptServer + 'runtest.php?url=http%3A%2F%2Ftwitter.com%2Fmarcelduran&location=Local_Firefox_Chrome%3AChrome&runs=3&fvonly=1&label=test%20123&timeline=1&netlog=1&wappalyzerPR=1&pngss=1&f=json'); done(); }); }); @@ -352,4 +353,16 @@ describe('WebPageTest Command Line', function() { }); }); + it('gets a test with long custom metrics script then returns API url and payload with custom metrics data', function (done) { + let script = '"[example]\n\\\\' + 'X'.repeat(6000) + '\nreturn 1;"' + + exec(mock('test http://foobar.com --http_method POST --custom ' + script), function (err, data) { + if (err) return done(err); + data = JSON.parse(data); + assert.equal(data.url, wptServer + 'runtest.php'); + assert.equal(data.form.length, 6077); + done(); + }); + }); + }); diff --git a/test/edge-cases-test.js b/test/edge-cases-test.js index 77ab4eb..653b31a 100644 --- a/test/edge-cases-test.js +++ b/test/edge-cases-test.js @@ -85,6 +85,30 @@ describe('Edge Cases of', function() { }); }); + it('gets a test with long custom metrics script then returns API url and payload with custom metrics data', function (done) { + wpt.runTest('http://foobar.com', { + dryRun: true, + mobile: 1, + http_method: 'POST', + custom: '[example]\n\\\\' + 'X'.repeat(10000) + '\nreturn 1;' + }, function (err, data) { + if (err) return done(err); + assert.equal(data.url, wptServer + 'runtest.php'); + assert.equal(data.form.length, 10089); + done(); + }); + }); + + it('gets a test with custom wappalyzer rules then returns API url', function (done) { + wpt.runTest('http://foobar.com', { + dryRun: true, + wappalyzerpr: 1, + }, function (err, data) { + if (err) return done(err); + assert.equal(data.url, wptServer + 'runtest.php?url=http%3A%2F%2Ffoobar.com&wappalyzerPR=1&f=json'); + done(); + }); + }); }); describe('WebPageTest localhost helper', function() { diff --git a/test/fixtures/command-line/help-test.txt b/test/fixtures/command-line/help-test.txt index ed98f04..15d6e64 100644 --- a/test/fixtures/command-line/help-test.txt +++ b/test/fixtures/command-line/help-test.txt @@ -60,7 +60,7 @@ Options: switches (Chrome only) --lighthouse perform lighthouse test (Chrome only, Linux agent only) - -thc, --throttleCPU custom cpu throttling + -thc, --throttleCPU custom cpu throttling -g, --login username for authenticating tests (http authentication) -w, --password password for authenticating tests (http @@ -78,6 +78,8 @@ Options: drop all requests -c, --custom