Skip to content

Adjusting API request to POST if too much data #178

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

Adjusting API request to POST if too much data #178

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
27d3353
conditional post requests + test
max-ostapenko Aug 1, 2023
daacc92
deduplicated get and post functions
max-ostapenko Aug 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 46 additions & 32 deletions lib/helper.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,24 +5,25 @@
* Released under the MIT License
*/

var xml2js = require('xml2js'),
url = require('url'),
os = require('os'),
csv = require('csv'),
entities = require('entities');
var xml2js = require('xml2js'),
url = require('url'),
os = require('os'),
csv = require('csv'),
entities = require('entities'),
qs = require('querystring');

var parser = new xml2js.Parser({explicitArray: false, mergeAttrs: true});
var parser = new xml2js.Parser({ explicitArray: false, mergeAttrs: true });

var reNumber = /^[\.\+\-]?[\d\.]+$/,
reInvalidDec = /(?:\.\d*){2,}/,
reDec = /\./,
reLineBreak = /[\n\r]+/g,
reLastBreak = /\n$/,
reProtocol = /^https?:\/\//i,
reIp = /\d+\.\d+\.\d+\.\d+/, // 127.0.0.1
reInvalidDec = /(?:\.\d*){2,}/,
reDec = /\./,
reLineBreak = /[\n\r]+/g,
reLastBreak = /\n$/,
reProtocol = /^https?:\/\//i,
reIp = /\d+\.\d+\.\d+\.\d+/, // 127.0.0.1

TAB = '\t',
NEWLINE = '\n';
TAB = '\t',
NEWLINE = '\n';

function parseNumber(s) {
if (typeof s !== 'string' || !reNumber.test(s) || reInvalidDec.test(s)) {
Expand All @@ -34,7 +35,7 @@ function parseNumber(s) {

function normalizeObj(root) {
if (typeof root === 'object') {
Object.keys(root).forEach(function(key) {
Object.keys(root).forEach(function (key) {
var value = root[key];
if (typeof value === 'string') {
if (value.length === 0 || value === '\n') {
Expand Down Expand Up @@ -62,8 +63,8 @@ function xmlToObj(xml, callback) {

function svToObj(delimiter, headers, sv) {
var data,
start = 0,
obj = {};
start = 0,
obj = {};

delimiter = delimiter || ',';

Expand Down Expand Up @@ -101,11 +102,11 @@ function svToObj(delimiter, headers, sv) {
}

function csvParser(data, callback) {
csv.parse(data.toString(), { columns: true }, function(err, data) {
csv.parse(data.toString(), { columns: true }, function (err, data) {
if (err) {
callback.bind(this, err);
}
csv.transform(data, function(row) {
csv.transform(data, function (row) {
var key, value;
for (key in row) {
value = row[key].replace(/<b>|<\/b>/g, '');
Expand Down Expand Up @@ -155,19 +156,32 @@ function scriptToString(data) {
}

// Build the RESTful API url call only
function dryRun(config, path) {
function dryRun(config, path, form) {
path = url.parse(path, true);

return {
url: url.format({
protocol: config.protocol,
hostname: config.hostname,
port: (config.port !== 80 && config.port !== 443 ?
config.port : undefined),
pathname: path.pathname,
query: path.query
})
};
if (config.method == "POST") {
return {
url: url.format({
protocol: config.protocol,
hostname: config.hostname,
port: (config.port !== 80 && config.port !== 443 ?
config.port : undefined),
pathname: path.pathname,
}),
form: qs.stringify(form)
};
} else {
return {
url: url.format({
protocol: config.protocol,
hostname: config.hostname,
port: (config.port !== 80 && config.port !== 443 ?
config.port : undefined),
pathname: path.pathname,
query: path.query
})
};
}
}

// Normalize server config
Expand Down Expand Up @@ -221,8 +235,8 @@ function setQuery(map, options, query) {
map.options.forEach(function eachOpts(opt) {
Object.keys(opt).forEach(function eachOpt(key) {
var param = opt[key],
name = param.name,
value = options[name] || options[key];
name = param.name,
value = options[name] || options[key];

if (value !== undefined && param.api) {
if (param.array) {
Expand Down
46 changes: 36 additions & 10 deletions lib/webpagetest.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ var http = require("http"),
specs = require("./specs"),
helper = require("./helper"),
server = require("./server"),
mapping = require("./mapping");
mapping = require("./mapping"),
qs = require('querystring');

var reSpace = /\s/,
reConnectivity =
Expand Down Expand Up @@ -57,8 +58,8 @@ var filenames = {
cached: "_Cached",
};

// GET helper function
function get(config, pathname, proxy, agent, callback, encoding) {
// GET/POST helper function
function get(config, pathname, data, proxy, agent, callback, encoding) {
var protocol, options;

if (proxy) {
Expand All @@ -79,6 +80,8 @@ function get(config, pathname, proxy, agent, callback, encoding) {
headers: {
Host: config.hostname,
},
method: config.method

};
} else {
protocol = config.protocol === "https:" ? https : http;
Expand All @@ -87,10 +90,15 @@ function get(config, pathname, proxy, agent, callback, encoding) {
host: config.hostname,
auth: config.auth,
port: config.port,
method: config.method,
headers: {},
};
}

if (options.method == "POST") {
options.headers['Content-Type'] = 'application/x-www-form-urlencoded';
}

if (encoding !== "binary") {
options.headers["X-WPT-API-KEY"] = this.config.key;
options.headers["accept-encoding"] = "gzip,deflate";
Expand All @@ -101,8 +109,8 @@ function get(config, pathname, proxy, agent, callback, encoding) {
options.agent = agent;
}

return protocol
.get(options, function getResponse(res) {
var request = protocol
.request(options, function getResponse(res) {
var data,
length,
statusCode = res.statusCode;
Expand Down Expand Up @@ -158,7 +166,14 @@ function get(config, pathname, proxy, agent, callback, encoding) {
})
.on("error", function onError(err) {
callback(err);
});
})

if (options.method == "POST") {
return request.end(qs.stringify(data));
} else {
return request.end();
}

}

// execute callback properly normalizing optional args
Expand Down Expand Up @@ -186,22 +201,33 @@ function api(pathname, callback, query, options) {
config = this.config;
}

pathname = url.format({
pathname: url.resolve(config.pathname, pathname),
pathname = url.resolve(config.pathname, pathname);

config.method = url.format({
pathname: pathname,
query: query,
});
}).toString().length > 6 * 1024 ? "POST" : "GET";

if (config.method == "GET") {
pathname = url.format({
pathname: pathname,
query: query,
});
query = undefined;
}

if (options.dryRun) {
// dry run: return the API url (string) only
if (typeof callback === "function") {
callback.apply(callback, [undefined, helper.dryRun(config, pathname)]);
callback.apply(callback, [undefined, helper.dryRun(config, pathname, query)]);
}
} else {
// make the real API call
get.call(
this,
config,
pathname,
query,
options.proxy,
options.agent,
function apiCallback(err, data, info) {
Expand Down
13 changes: 13 additions & 0 deletions test/edge-cases-test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,19 @@ describe('Edge Cases of', function() {
});
});

it('gets a test with custom metrics then returns API url and payload with custom metrics data present', function (done) {
wpt.runTest('http://foobar.com', {
dryRun: true,
mobile: 1,
custom: '[example]\n\\\\' + 'X'.repeat(6 * 1024) + '\nreturn 1;'
}, function (err, data) {
if (err) return done(err);
assert.equal(data.url, wptServer + 'runtest.php');
assert.equal(data.form.length, 6233);
done();
});
});

});

describe('WebPageTest localhost helper', function() {
Expand Down