martin/stone
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

added server caching

Browse Source
This commit is contained in:
Martin Donnelly 2017-09-11 15:29:31 +01:00
parent 6ce99d1dac
commit 51dc4a5beb
4 changed files with 167 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
package-lock.json generated
View File

@ -39,7 +39,6 @@
"version": "4.11.8",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-4.11.8.tgz",
"integrity": "sha1-gv+wKynmYq5TvcIK8VlHcGc5xTY=",
"optional": true,
"requires": {
"co": "4.6.0",
"json-stable-stringify": "1.0.1"
@ -56,6 +55,11 @@
"integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
"optional": true
},
"apicache": {
"version": "0.11.2",
"resolved": "https://registry.npmjs.org/apicache/-/apicache-0.11.2.tgz",
"integrity": "sha512-olsFYz/fxxJPnZ3I8KG9CL40PaibfWhXJ+C0JO2ti4N3xsnCJ3plm50tj4eBguMqHaUBBVqsaFGyfZ8vpyZXEg=="
},
"array-flatten": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
@ -64,14 +68,12 @@
"asn1": {
"version": "0.2.3",
"resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz",
"integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y=",
"optional": true
"integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y="
},
"assert-plus": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz",
"integrity": "sha1-104bh+ev/A24qttwIfP+SBAasjQ=",
"optional": true
"integrity": "sha1-104bh+ev/A24qttwIfP+SBAasjQ="
},
"ast-types": {
"version": "0.9.12",
@ -91,20 +93,17 @@
"asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=",
"optional": true
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
},
"aws-sign2": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz",
"integrity": "sha1-FDQt0428yU0OW4fXY81jYSwOeU8=",
"optional": true
"integrity": "sha1-FDQt0428yU0OW4fXY81jYSwOeU8="
},
"aws4": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.6.0.tgz",
"integrity": "sha1-g+9cqGCysy5KDe7e6MdxudtXRx4=",
"optional": true
"integrity": "sha1-g+9cqGCysy5KDe7e6MdxudtXRx4="
},
"axios": {
"version": "0.15.3",
@ -181,8 +180,7 @@
"caseless": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=",
"optional": true
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
},
"chalk": {
"version": "1.1.3",
@ -200,8 +198,7 @@
"co": {
"version": "4.6.0",
"resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
"integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=",
"optional": true
"integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ="
},
"combined-stream": {
"version": "1.0.5",
@ -246,7 +243,6 @@
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz",
"integrity": "sha1-O9/s3GCBR8HGcgL6KR59ylnqo7g=",
"optional": true,
"requires": {
"boom": "2.10.1"
}
@ -255,7 +251,6 @@
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
"integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=",
"optional": true,
"requires": {
"assert-plus": "1.0.0"
},
@ -263,8 +258,7 @@
"assert-plus": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
"optional": true
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
}
}
},
@ -473,14 +467,12 @@
"forever-agent": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
"integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=",
"optional": true
"integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE="
},
"form-data": {
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-2.1.4.tgz",
"integrity": "sha1-M8GDrPGTJ27KqYFDpp6Uv+4XUNE=",
"optional": true,
"requires": {
"asynckit": "0.4.0",
"combined-stream": "1.0.5",
@ -560,7 +552,6 @@
"version": "0.1.7",
"resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
"integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
"optional": true,
"requires": {
"assert-plus": "1.0.0"
},
@ -568,22 +559,19 @@
"assert-plus": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
"optional": true
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
}
}
},
"har-schema": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/har-schema/-/har-schema-1.0.5.tgz",
"integrity": "sha1-0mMTX0MwfALGAq/I/pWXDAFRNp4=",
"optional": true
"integrity": "sha1-0mMTX0MwfALGAq/I/pWXDAFRNp4="
},
"har-validator": {
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/har-validator/-/har-validator-4.2.1.tgz",
"integrity": "sha1-M0gdDxu/9gDdID11gSpqX7oALio=",
"optional": true,
"requires": {
"ajv": "4.11.8",
"har-schema": "1.0.5"
@ -602,7 +590,6 @@
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz",
"integrity": "sha1-B4REvXwWQLD+VA0sm3PVlnjo4cQ=",
"optional": true,
"requires": {
"boom": "2.10.1",
"cryptiles": "2.0.5",
@ -655,7 +642,6 @@
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz",
"integrity": "sha1-33LiZwZs0Kxn+3at+OE0qPvPkb8=",
"optional": true,
"requires": {
"assert-plus": "0.2.0",
"jsprim": "1.4.1",
@ -745,8 +731,7 @@
"is-typedarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
"integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=",
"optional": true
"integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo="
},
"isarray": {
"version": "1.0.0",
@ -756,8 +741,7 @@
"isstream": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
"integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=",
"optional": true
"integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
},
"jsbn": {
"version": "0.1.1",
@ -768,14 +752,12 @@
"json-schema": {
"version": "0.2.3",
"resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
"integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=",
"optional": true
"integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM="
},
"json-stable-stringify": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz",
"integrity": "sha1-mnWdOcXy/1A/1TAGRu1EX4jE+a8=",
"optional": true,
"requires": {
"jsonify": "0.0.0"
}
@ -788,8 +770,7 @@
"jsonify": {
"version": "0.0.0",
"resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz",
"integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM=",
"optional": true
"integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM="
},
"jsonpointer": {
"version": "4.0.1",
@ -801,7 +782,6 @@
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
"integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=",
"optional": true,
"requires": {
"assert-plus": "1.0.0",
"extsprintf": "1.3.0",
@ -812,8 +792,7 @@
"assert-plus": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
"optional": true
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
}
}
},
@ -1068,6 +1047,14 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
},
"nano-cache": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/nano-cache/-/nano-cache-1.1.1.tgz",
"integrity": "sha1-UQbZXzkstENZZU4hzAFa4hjnVGY=",
"requires": {
"extend": "3.0.1"
}
},
"negotiator": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz",
@ -1165,8 +1152,7 @@
"oauth-sign": {
"version": "0.8.2",
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz",
"integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=",
"optional": true
"integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM="
},
"on-finished": {
"version": "2.3.0",
@ -1267,8 +1253,7 @@
"performance-now": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/performance-now/-/performance-now-0.2.0.tgz",
"integrity": "sha1-M+8wxcd9TqIcWlOGnZG1bY8lVeU=",
"optional": true
"integrity": "sha1-M+8wxcd9TqIcWlOGnZG1bY8lVeU="
},
"pinkie": {
"version": "2.0.4",
@ -1328,8 +1313,7 @@
"punycode": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
"integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=",
"optional": true
"integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
},
"q": {
"version": "1.4.1",
@ -1404,7 +1388,6 @@
"version": "2.81.0",
"resolved": "https://registry.npmjs.org/request/-/request-2.81.0.tgz",
"integrity": "sha1-xpKJRqDgbF+Nb4qTM0af/aRimKA=",
"optional": true,
"requires": {
"aws-sign2": "0.6.0",
"aws4": "1.6.0",
@ -1433,11 +1416,19 @@
"qs": {
"version": "6.4.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.4.0.tgz",
"integrity": "sha1-E+JtKK1rD/qpExLNO/cI7TUecjM=",
"optional": true
"integrity": "sha1-E+JtKK1rD/qpExLNO/cI7TUecjM="
}
}
},
"request-promise-cache": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/request-promise-cache/-/request-promise-cache-1.0.6.tgz",
"integrity": "sha1-d8opARFlG0UnLzFA6cDoNXCHiMM=",
"requires": {
"nano-cache": "1.1.1",
"request": "2.81.0"
}
},
"requestretry": {
"version": "1.12.2",
"resolved": "https://registry.npmjs.org/requestretry/-/requestretry-1.12.2.tgz",
@ -1523,7 +1514,6 @@
"version": "1.0.9",
"resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz",
"integrity": "sha1-ZUEYTMkK7qbG57NeJlkIJEPGYZg=",
"optional": true,
"requires": {
"hoek": "2.16.3"
}
@ -1564,7 +1554,6 @@
"version": "1.13.1",
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.13.1.tgz",
"integrity": "sha1-US322mKHFEMW3EwY/hzx2UBzm+M=",
"optional": true,
"requires": {
"asn1": "0.2.3",
"assert-plus": "1.0.0",
@ -1579,8 +1568,7 @@
"assert-plus": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
"optional": true
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
}
}
},
@ -1632,8 +1620,7 @@
"stringstream": {
"version": "0.0.5",
"resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz",
"integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg=",
"optional": true
"integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg="
},
"strip-ansi": {
"version": "3.0.1",
@ -1666,7 +1653,6 @@
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.2.tgz",
"integrity": "sha1-8IH3bkyFcg5sN6X6ztc3FQ2EByo=",
"optional": true,
"requires": {
"punycode": "1.4.1"
}
@ -1681,7 +1667,6 @@
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
"integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
"optional": true,
"requires": {
"safe-buffer": "5.1.1"
}
@ -1763,8 +1748,7 @@
"uuid": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz",
"integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==",
"optional": true
"integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g=="
},
"vary": {
"version": "1.1.1",
@ -1775,7 +1759,6 @@
"version": "1.10.0",
"resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
"integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=",
"optional": true,
"requires": {
"assert-plus": "1.0.0",
"core-util-is": "1.0.2",
@ -1785,8 +1768,7 @@
"assert-plus": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
"optional": true
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
}
}
},

2
package.json
View File

@ -9,12 +9,14 @@
"author": "",
"license": "ISC",
"dependencies": {
"apicache": "^0.11.2",
"body-parser": "^1.18.0",
"express": "^4.15.4",
"http": "0.0.0",
"https": "^1.0.0",
"log4js": "^2.3.3",
"path": "^0.12.7",
"request-promise-cache": "^1.0.6",
"url": "^0.11.0"
}
}

115
server-cache.js Normal file
View File

@ -0,0 +1,115 @@
const express = require('express');
const app = express();
const path = require('path');
const bodyParser = require('body-parser');
const config = require('./config');
const log4js = require('log4js');
const logger = log4js.getLogger();
const URL = require('url');
const http = require('http');
const https = require('https');
const apicache = require('apicache');
logger.level = 'debug';
// app.use(compression());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ 'extended': true }));
const cache = apicache.middleware;
app.use(cache('5 minutes'));
function bob(req, res) {
logger.debug(req.params);
res.sendStatus(200);
}
function getUrl (req, res) {
const theUrl = req.params.encoded_id;
logger.debug('Want', theUrl);
const options = URL.parse(theUrl);
options.followAllRedirects = true;
options.headers = {
'User-Agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
};
// console.log('Options', options);
logger.info('>> getting url');
function urlQuery (callback) {
try {
let count = 0;
http.request(options, responseHandler).end();
function responseHandler(response) {
console.log(`STATUS: ${response.statusCode}`);
console.log(`HEADERS: ${JSON.stringify(response.headers)}`);
response.setEncoding('utf8');
if (response.statusCode === 302 || response.statusCode === 301) {
body = [];
const rUrl = URL.parse(response.headers.location);
rUrl.followAllRedirects = true;
rUrl.headers = {
'User-Agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
};
logger.debug('>> follow', response.headers.location, count);
// logger.debug(rUrl);
count++;
if (rUrl.protocol === 'https:')
https.request(rUrl, responseHandler).end();
else
http.request(rUrl, responseHandler).end();
// this.request(response.headers.location);
// console.log(this);
}
let data = '';
response.on('data', chunk => {
data += chunk;
});
response.on('end', () => {
if (response.statusCode !== 302 && response.statusCode !== 301)
callback(data);
});
response.on('error', e => {
logger.error(e);
});
}
/* http.request(options).on('response', response => {
let data = '';
response.on('data', chunk => {
data += chunk;
});
response.on('end', () => {
callback(data);
});
response.on('error', e => {
logger.error(e);
});
}).end();*/
}
catch (e) {
logger.error(e);
}
}
urlQuery(a => {
// logger.info(a);
logger.info('Got result');
// res.setHeader('Content-Type', 'application/json');
res.send(a);
});
}
app.get('/:encoded_id', getUrl);
const server = app.listen(config.port, () => {
logger.info(`Server listening on port ${config.port}`);
});

4
server.js
View File

@ -10,12 +10,16 @@ const logger = log4js.getLogger();
const URL = require('url');
const http = require('http');
const https = require('https');
const apicache = require('apicache');
logger.level = 'debug';
// app.use(compression());
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ 'extended': true }));
const cache = apicache.middleware;
app.use(cache('5 minutes'));
function bob(req, res) {
logger.debug(req.params);