sync

README.md

@@ -98,6 +98,7 @@ webpagetest --help
 - **-s, --server** _\<server\>_: the WPT server URL [https://www.webpagetest.org]
 - **-d, --dryrun**: just return the RESTful API URL
 - **-o, --out** _\<file\>_: place the output into \<file\>. Defaults to stdout
+- **--http_method** _\<method\>_: the HTTP method to use (GET, POST) [GET]
 
 _The default WPT server can also be specified via environment variable `WEBPAGETEST_SERVER`_
 
@@ -139,6 +140,7 @@ _The default WPT server can also be specified via environment variable `WEBPAGET
 - **-b, --block** _\<urls\>_: space-delimited list of urls to block (substring match)
 - **-Z, --spof** _\<domains\>_: space-delimited list of domains to simulate failure by re-routing to blackhole.webpagetest.org to silently drop all requests
 - **-c, --custom** _\<script\>_: execute arbitrary javascript at the end of a test to collect custom metrics
+- **-w, --wappalyzerpr** _\<number\>_: set the wappalyzer fork PR number to use for technology detection
 - **-a, --authtype** _\<type\>_: type of authentication: 0 = Basic, 1 = SNS [0]
 - **-n, --notify** _\<e-mail\>_: e-mail address to notify with the test results
 - **-B, --pingback** _\<url\>_: URL to ping when the test is complete (the test ID will be passed as an "id" parameter)
@@ -316,7 +318,7 @@ webpagetest status 121025_PT_N8K -k YOURAPIKEY
 #### 4. Get test results
 
 ```bash
-webpagetest results 121025_PT_N8K -k YOURAPIKEY 
+webpagetest results 121025_PT_N8K -k YOURAPIKEY
 ```
 
 ```javascript
@@ -480,6 +482,7 @@ wpt.runTest(script, (err, data) => {
 
 - **dryRun**: _Boolean_, if `true`, method does not make an actual request to the API Server but rather returns an object with `url` which contains the actual URL to make the GET request to WebPageTest API Server
 - **server**: _String_, if specified, overrides the WebPageTest server informed in the constructor only for that method call
+- **http_method**: _String_, if specified, overrides the HTTP method in the constructor only for that method call (GET, POST) [GET]
 
 #### Test (works with `runTest` and `runTestAndWait`)
 
@@ -517,6 +520,7 @@ wpt.runTest(script, (err, data) => {
 - **block**: _String_, space-delimited list of urls to block (substring match)
 - **spof**: _String_, space-delimited list of domains to simulate failure by re-routing to blackhole.webpagetest.org to silently drop all requests
 - **customMetrics**: _String_, execute arbitrary JavaScript at the end of a test to collect custom metrics
+- **wappalyzerpr** _Integer_, set the wappalyzer fork PR number to use for technology detection
 - **authenticationType**: _Number_, type of authentication: 0 = Basic, 1 = SNS [0]
 - **notifyEmail**: _String_, e-mail address to notify with the test results
 - **pingback**: _String_, URL to ping when the test is complete (the test ID will be passed as an "id" parameter)

bin/webpagetest

@@ -269,6 +269,7 @@ function cleanupProgram() {
   delete program.server;
   delete program.dryrun;
   delete program.out;
+  delete program.http_method;
 }
 
 function execCommand() {
@@ -277,6 +278,7 @@ function execCommand() {
   // options
   if (options) {
     options.dryRun = program.dryrun;
+    options.http_method = program.http_method;
     args.push(options);
   }
 

lib/helper.js

@@ -9,7 +9,8 @@ var xml2js   = require('xml2js'),
     url      = require('url'),
     os       = require('os'),
     csv      = require('csv'),
-    entities = require('entities');
+    entities = require('entities')
+    qs       = require('querystring');
 
 var parser = new xml2js.Parser({explicitArray: false, mergeAttrs: true});
 
@@ -155,9 +156,22 @@ function scriptToString(data) {
 }
 
 // Build the RESTful API url call only
-function dryRun(config, path) {
+function dryRun(config, path, form) {
   path = url.parse(path, true);
 
+  if (config.method == "POST") {
+    return {
+      url: url.format({
+        protocol: config.protocol,
+        hostname: config.hostname,
+        port: (config.port !== 80 && config.port !== 443 ?
+          config.port : undefined),
+        pathname: path.pathname,
+      }),
+      form: qs.stringify(form)
+    };
+  }
+
   return {
     url: url.format({
       protocol: config.protocol,

lib/mapping.js

@@ -21,6 +21,11 @@ var options = {
       bool: true,
       info: "just return the RESTful API URL",
     },
+    http_method: {
+      name: "http_method",
+      param: "string",
+      info: "HTTP method to use for submitting the test (GET or POST) [GET]",
+    }
   },
   test: {
     location: {
@@ -307,6 +312,14 @@ var options = {
       info: "execute arbitrary JavaScript at the end of a test to collect custom metrics",
     },
 
+    wappalyzerpr: {
+      name: "wappalyzerpr",
+      key: "wpr",
+      api: "wappalyzerPR",
+      param: "number",
+      info: "PR number of the Wappalyzer fork to use in technology detection",
+    },
+
     // API only settings
     authtype: {
       name: "authenticationType",
@@ -836,6 +849,13 @@ var options = {
       param: "script",
       info: "execute arbitrary JavaScript at the end of a test to collect custom metrics",
     },
+    wappalyzerpr: {
+      name: "wappalyzerpr",
+      key: "wpr",
+      api: "wappalyzerPR",
+      param: "number",
+      info: "PR number of the Wappalyzer fork to use in technology detection",
+    },
 
     // API only settings
     authtype: {

lib/webpagetest.js

@@ -13,12 +13,14 @@ var http = require("http"),
   specs = require("./specs"),
   helper = require("./helper"),
   server = require("./server"),
-  mapping = require("./mapping");
+  mapping = require("./mapping"),
+  qs = require('querystring');
 
 var reSpace = /\s/,
   reConnectivity =
     /^(?:Cable|DSL|3GSlow|3G|3GFast|4G|LTE|Edge|2G|Dial|FIOS|Native|custom)$/,
-  reHTMLOutput = /<h\d[^<]*>([^<]+)<\/h\d>/; // for H3 on cancelTest.php
+  reHTMLOutput = /<h\d[^<]*>([^<]+)<\/h\d>/, // for H3 on cancelTest.php
+  reHTTPmethods = /^(?:GET|POST)$/;
 
 var paths = {
   testStatus: "testStatus.php",
@@ -57,8 +59,8 @@ var filenames = {
   cached: "_Cached",
 };
 
-// GET helper function
-function get(config, pathname, proxy, agent, callback, encoding) {
+// GET/POST helper function
+function get(config, pathname, data, proxy, agent, callback, encoding) {
   var protocol, options;
 
   if (proxy) {
@@ -79,6 +81,7 @@ function get(config, pathname, proxy, agent, callback, encoding) {
       headers: {
         Host: config.hostname,
       },
+      method: config.method
     };
   } else {
     protocol = config.protocol === "https:" ? https : http;
@@ -87,10 +90,15 @@ function get(config, pathname, proxy, agent, callback, encoding) {
       host: config.hostname,
       auth: config.auth,
       port: config.port,
+      method: config.method,
       headers: {},
     };
   }
 
+  if (options.method == "POST") {
+    options.headers['Content-Type'] = 'application/x-www-form-urlencoded';
+  }
+
   // api key always required
   options.headers["X-WPT-API-KEY"] = this.config.key;
   options.headers["accept-encoding"] = "gzip,deflate";
@@ -100,8 +108,8 @@ function get(config, pathname, proxy, agent, callback, encoding) {
     options.agent = agent;
   }
 
-  return protocol
-    .get(options, function getResponse(res) {
+  var request = protocol
+    .request(options, function getResponse(res) {
       var data,
         length,
         statusCode = res.statusCode;
@@ -158,6 +166,13 @@ function get(config, pathname, proxy, agent, callback, encoding) {
     .on("error", function onError(err) {
       callback(err);
     });
+
+  if (options.method == "POST") {
+    return request.end(qs.stringify(data));
+  }
+
+  return request.end();
+
 }
 
 // execute callback properly normalizing optional args
@@ -185,22 +200,35 @@ function api(pathname, callback, query, options) {
     config = this.config;
   }
 
-  pathname = url.format({
-    pathname: url.resolve(config.pathname, pathname),
-    query: query,
-  });
+  pathname = url.resolve(config.pathname, pathname);
+
+  if (reHTTPmethods.test(options.http_method)) {
+    config.method = options.http_method;
+  } else {
+    config.method = WebPageTest.defaultHTTPMethod;
+  }
+
+
+  if (config.method == "GET") {
+    pathname = url.format({
+      pathname: pathname,
+      query: query,
+    });
+    query = undefined;
+  }
 
   if (options.dryRun) {
     // dry run: return the API url (string) only
     if (typeof callback === "function") {
-      callback.apply(callback, [undefined, helper.dryRun(config, pathname)]);
+      callback.apply(callback, [undefined, helper.dryRun(config, pathname, query)]);
     }
   } else {
     // make the real API call
     get.call(
       this,
       config,
       pathname,
+      query,
       options.proxy,
       options.agent,
       function apiCallback(err, data, info) {
@@ -931,6 +959,7 @@ WebPageTest.filenames = filenames;
 WebPageTest.defaultServer = "https://www.webpagetest.org";
 WebPageTest.defaultListenPort = 7791;
 WebPageTest.defaultWaitResultsPort = 8000;
+WebPageTest.defaultHTTPMethod = "GET";
 
 // Version
 Object.defineProperty(WebPageTest, "version", {

test/command-line-test.js

@@ -102,11 +102,12 @@ describe('WebPageTest Command Line', function() {
         '--first ' +
         '--timeline ' +
         '--netlog ' +
+        '--wappalyzerpr 1 ' +
         '--full'
       ), function(err, data) {
       if (err) return done(err);
       data = JSON.parse(data);
-      assert.equal(data.url, wptServer + 'runtest.php?url=http%3A%2F%2Ftwitter.com%2Fmarcelduran&location=Local_Firefox_Chrome%3AChrome&runs=3&fvonly=1&label=test%20123&timeline=1&netlog=1&pngss=1&f=json');
+      assert.equal(data.url, wptServer + 'runtest.php?url=http%3A%2F%2Ftwitter.com%2Fmarcelduran&location=Local_Firefox_Chrome%3AChrome&runs=3&fvonly=1&label=test%20123&timeline=1&netlog=1&wappalyzerPR=1&pngss=1&f=json');
       done();
     });
   });
@@ -352,4 +353,16 @@ describe('WebPageTest Command Line', function() {
     });
   });
 
+  it('gets a test with long custom metrics script then returns API url and payload with custom metrics data', function (done) {
+    let script = '"[example]\n\\\\' + 'X'.repeat(6000) + '\nreturn 1;"'
+
+    exec(mock('test http://foobar.com --http_method POST --custom ' + script), function (err, data) {
+      if (err) return done(err);
+      data = JSON.parse(data);
+      assert.equal(data.url, wptServer + 'runtest.php');
+      assert.equal(data.form.length, 6077);
+      done();
+    });
+  });
+
 });

test/edge-cases-test.js

@@ -85,6 +85,30 @@ describe('Edge Cases of', function() {
       });
     });
 
+    it('gets a test with long custom metrics script then returns API url and payload with custom metrics data', function (done) {
+      wpt.runTest('http://foobar.com', {
+        dryRun: true,
+        mobile: 1,
+        http_method: 'POST',
+        custom: '[example]\n\\\\' + 'X'.repeat(10000) + '\nreturn 1;'
+      }, function (err, data) {
+        if (err) return done(err);
+        assert.equal(data.url, wptServer + 'runtest.php');
+        assert.equal(data.form.length, 10089);
+        done();
+      });
+    });
+
+    it('gets a test with custom wappalyzer rules then returns API url', function (done) {
+      wpt.runTest('http://foobar.com', {
+        dryRun: true,
+        wappalyzerpr: 1,
+      }, function (err, data) {
+        if (err) return done(err);
+        assert.equal(data.url, wptServer + 'runtest.php?url=http%3A%2F%2Ffoobar.com&wappalyzerPR=1&f=json');
+        done();
+      });
+    });
   });
 
   describe('WebPageTest localhost helper', function() {

test/fixtures/command-line/help-test.txt

@@ -60,7 +60,7 @@ Options:
                                       switches (Chrome only)
       --lighthouse                    perform lighthouse test (Chrome only,
                                       Linux agent only)
-  -thc, --throttleCPU <number>        custom cpu throttling    
+  -thc, --throttleCPU <number>        custom cpu throttling
   -g, --login <username>              username for authenticating tests (http
                                       authentication)
   -w, --password <password>           password for authenticating tests (http
@@ -78,6 +78,8 @@ Options:
                                       drop all requests
   -c, --custom <script>               execute arbitrary JavaScript at the end
                                       of a test to collect custom metrics
+  -wpr, --wappalyzerpr <number>       set the wappalyzerPR fork PR number to
+                                      use for technology detection
   -a, --authtype <type>               type of authentication: 0 = Basic, 1 =
                                       SNS [0]
   -n, --notify <e-mail>               e-mail address to notify with the test
@@ -88,7 +90,7 @@ Options:
   -D, --bwdown <bandwidth>            download bandwidth in Kbps (used when
                                       specifying a custom connectivity profile)
   -U, --bwup <bandwidth>              upload bandwidth in Kbps (used when
-                                      specifying a custom connectivity profile)               
+                                      specifying a custom connectivity profile)
   -bw, --browserwidth <pixels>        Browser window width (in display pixels)
   -bh, --browserheight <pixels>       Browser window height (in display pixels)
   -vh, --viewportheight <pixels>      Viewport Height in css pixels

test/fixtures/command-line/help-testAndWait.txt

@@ -60,7 +60,7 @@ Options:
                                       switches (Chrome only)
       --lighthouse                    perform lighthouse test (Chrome only,
                                       Linux agent only)
-  -thc, --throttleCPU <number>        custom cpu throttling    
+  -thc, --throttleCPU <number>        custom cpu throttling
   -g, --login <username>              username for authenticating tests (http
                                       authentication)
   -w, --password <password>           password for authenticating tests (http
@@ -78,6 +78,8 @@ Options:
                                       drop all requests
   -c, --custom <script>               execute arbitrary JavaScript at the end
                                       of a test to collect custom metrics
+  -wpr, --wappalyzerpr <number>         set the wappalyzerPR fork PR number to
+                                      use for technology detection
   -a, --authtype <type>               type of authentication: 0 = Basic, 1 =
                                       SNS [0]
   -n, --notify <e-mail>               e-mail address to notify with the test
@@ -88,7 +90,7 @@ Options:
   -D, --bwdown <bandwidth>            download bandwidth in Kbps (used when
                                       specifying a custom connectivity profile)
   -U, --bwup <bandwidth>              upload bandwidth in Kbps (used when
-                                      specifying a custom connectivity profile)               
+                                      specifying a custom connectivity profile)
   -bw, --browserwidth <pixels>        Browser window width (in display pixels)
   -bh, --browserheight <pixels>       Browser window height (in display pixels)
   -vh, --viewportheight <pixels>      Viewport Height in css pixels

test/fixtures/command-line/help.txt

@@ -5,6 +5,8 @@ Options:
   -s, --server <server>             the WPT server URL
                                     [https://www.webpagetest.org]
   -d, --dryrun                      just return the RESTful API URL
+  --http_method <string>            HTTP method to use for submitting the test
+                                    (GET or POST) [GET]
   -o, --out <file>                  place the output into <file>. Defaults to
                                     stdout
   -h, --help                        display help for command