diff --git a/package-lock.json b/package-lock.json index 43619b2..de5e531 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,7 +39,6 @@ "version": "4.11.8", "resolved": "https://registry.npmjs.org/ajv/-/ajv-4.11.8.tgz", "integrity": "sha1-gv+wKynmYq5TvcIK8VlHcGc5xTY=", - "optional": true, "requires": { "co": "4.6.0", "json-stable-stringify": "1.0.1" @@ -56,6 +55,11 @@ "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=", "optional": true }, + "apicache": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/apicache/-/apicache-0.11.2.tgz", + "integrity": "sha512-olsFYz/fxxJPnZ3I8KG9CL40PaibfWhXJ+C0JO2ti4N3xsnCJ3plm50tj4eBguMqHaUBBVqsaFGyfZ8vpyZXEg==" + }, "array-flatten": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", @@ -64,14 +68,12 @@ "asn1": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz", - "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y=", - "optional": true + "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y=" }, "assert-plus": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz", - "integrity": "sha1-104bh+ev/A24qttwIfP+SBAasjQ=", - "optional": true + "integrity": "sha1-104bh+ev/A24qttwIfP+SBAasjQ=" }, "ast-types": { "version": "0.9.12", @@ -91,20 +93,17 @@ "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", - "optional": true + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" }, "aws-sign2": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz", - "integrity": "sha1-FDQt0428yU0OW4fXY81jYSwOeU8=", - "optional": true + "integrity": "sha1-FDQt0428yU0OW4fXY81jYSwOeU8=" }, "aws4": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.6.0.tgz", - "integrity": "sha1-g+9cqGCysy5KDe7e6MdxudtXRx4=", - "optional": true + "integrity": "sha1-g+9cqGCysy5KDe7e6MdxudtXRx4=" }, "axios": { "version": "0.15.3", @@ -181,8 +180,7 @@ "caseless": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=", - "optional": true + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" }, "chalk": { "version": "1.1.3", @@ -200,8 +198,7 @@ "co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", - "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=", - "optional": true + "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=" }, "combined-stream": { "version": "1.0.5", @@ -246,7 +243,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz", "integrity": "sha1-O9/s3GCBR8HGcgL6KR59ylnqo7g=", - "optional": true, "requires": { "boom": "2.10.1" } @@ -255,7 +251,6 @@ "version": "1.14.1", "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", - "optional": true, "requires": { "assert-plus": "1.0.0" }, @@ -263,8 +258,7 @@ "assert-plus": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", - "optional": true + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" } } }, @@ -473,14 +467,12 @@ "forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", - "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=", - "optional": true + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" }, "form-data": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.1.4.tgz", "integrity": "sha1-M8GDrPGTJ27KqYFDpp6Uv+4XUNE=", - "optional": true, "requires": { "asynckit": "0.4.0", "combined-stream": "1.0.5", @@ -560,7 +552,6 @@ "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", - "optional": true, "requires": { "assert-plus": "1.0.0" }, @@ -568,22 +559,19 @@ "assert-plus": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", - "optional": true + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" } } }, "har-schema": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-1.0.5.tgz", - "integrity": "sha1-0mMTX0MwfALGAq/I/pWXDAFRNp4=", - "optional": true + "integrity": "sha1-0mMTX0MwfALGAq/I/pWXDAFRNp4=" }, "har-validator": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-4.2.1.tgz", "integrity": "sha1-M0gdDxu/9gDdID11gSpqX7oALio=", - "optional": true, "requires": { "ajv": "4.11.8", "har-schema": "1.0.5" @@ -602,7 +590,6 @@ "version": "3.1.3", "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz", "integrity": "sha1-B4REvXwWQLD+VA0sm3PVlnjo4cQ=", - "optional": true, "requires": { "boom": "2.10.1", "cryptiles": "2.0.5", @@ -655,7 +642,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz", "integrity": "sha1-33LiZwZs0Kxn+3at+OE0qPvPkb8=", - "optional": true, "requires": { "assert-plus": "0.2.0", "jsprim": "1.4.1", @@ -745,8 +731,7 @@ "is-typedarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", - "optional": true + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" }, "isarray": { "version": "1.0.0", @@ -756,8 +741,7 @@ "isstream": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=", - "optional": true + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" }, "jsbn": { "version": "0.1.1", @@ -768,14 +752,12 @@ "json-schema": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=", - "optional": true + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" }, "json-stable-stringify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz", "integrity": "sha1-mnWdOcXy/1A/1TAGRu1EX4jE+a8=", - "optional": true, "requires": { "jsonify": "0.0.0" } @@ -788,8 +770,7 @@ "jsonify": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz", - "integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM=", - "optional": true + "integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM=" }, "jsonpointer": { "version": "4.0.1", @@ -801,7 +782,6 @@ "version": "1.4.1", "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", - "optional": true, "requires": { "assert-plus": "1.0.0", "extsprintf": "1.3.0", @@ -812,8 +792,7 @@ "assert-plus": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", - "optional": true + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" } } }, @@ -1068,6 +1047,14 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, + "nano-cache": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/nano-cache/-/nano-cache-1.1.1.tgz", + "integrity": "sha1-UQbZXzkstENZZU4hzAFa4hjnVGY=", + "requires": { + "extend": "3.0.1" + } + }, "negotiator": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", @@ -1165,8 +1152,7 @@ "oauth-sign": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz", - "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=", - "optional": true + "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=" }, "on-finished": { "version": "2.3.0", @@ -1267,8 +1253,7 @@ "performance-now": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-0.2.0.tgz", - "integrity": "sha1-M+8wxcd9TqIcWlOGnZG1bY8lVeU=", - "optional": true + "integrity": "sha1-M+8wxcd9TqIcWlOGnZG1bY8lVeU=" }, "pinkie": { "version": "2.0.4", @@ -1328,8 +1313,7 @@ "punycode": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=", - "optional": true + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" }, "q": { "version": "1.4.1", @@ -1404,7 +1388,6 @@ "version": "2.81.0", "resolved": "https://registry.npmjs.org/request/-/request-2.81.0.tgz", "integrity": "sha1-xpKJRqDgbF+Nb4qTM0af/aRimKA=", - "optional": true, "requires": { "aws-sign2": "0.6.0", "aws4": "1.6.0", @@ -1433,11 +1416,19 @@ "qs": { "version": "6.4.0", "resolved": "https://registry.npmjs.org/qs/-/qs-6.4.0.tgz", - "integrity": "sha1-E+JtKK1rD/qpExLNO/cI7TUecjM=", - "optional": true + "integrity": "sha1-E+JtKK1rD/qpExLNO/cI7TUecjM=" } } }, + "request-promise-cache": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/request-promise-cache/-/request-promise-cache-1.0.6.tgz", + "integrity": "sha1-d8opARFlG0UnLzFA6cDoNXCHiMM=", + "requires": { + "nano-cache": "1.1.1", + "request": "2.81.0" + } + }, "requestretry": { "version": "1.12.2", "resolved": "https://registry.npmjs.org/requestretry/-/requestretry-1.12.2.tgz", @@ -1523,7 +1514,6 @@ "version": "1.0.9", "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz", "integrity": "sha1-ZUEYTMkK7qbG57NeJlkIJEPGYZg=", - "optional": true, "requires": { "hoek": "2.16.3" } @@ -1564,7 +1554,6 @@ "version": "1.13.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.13.1.tgz", "integrity": "sha1-US322mKHFEMW3EwY/hzx2UBzm+M=", - "optional": true, "requires": { "asn1": "0.2.3", "assert-plus": "1.0.0", @@ -1579,8 +1568,7 @@ "assert-plus": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", - "optional": true + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" } } }, @@ -1632,8 +1620,7 @@ "stringstream": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", - "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg=", - "optional": true + "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg=" }, "strip-ansi": { "version": "3.0.1", @@ -1666,7 +1653,6 @@ "version": "2.3.2", "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.2.tgz", "integrity": "sha1-8IH3bkyFcg5sN6X6ztc3FQ2EByo=", - "optional": true, "requires": { "punycode": "1.4.1" } @@ -1681,7 +1667,6 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", - "optional": true, "requires": { "safe-buffer": "5.1.1" } @@ -1763,8 +1748,7 @@ "uuid": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz", - "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==", - "optional": true + "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==" }, "vary": { "version": "1.1.1", @@ -1775,7 +1759,6 @@ "version": "1.10.0", "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", - "optional": true, "requires": { "assert-plus": "1.0.0", "core-util-is": "1.0.2", @@ -1785,8 +1768,7 @@ "assert-plus": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", - "optional": true + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" } } }, diff --git a/package.json b/package.json index bcc1426..1b6b816 100644 --- a/package.json +++ b/package.json @@ -9,12 +9,14 @@ "author": "", "license": "ISC", "dependencies": { + "apicache": "^0.11.2", "body-parser": "^1.18.0", "express": "^4.15.4", "http": "0.0.0", "https": "^1.0.0", "log4js": "^2.3.3", "path": "^0.12.7", + "request-promise-cache": "^1.0.6", "url": "^0.11.0" } } diff --git a/server-cache.js b/server-cache.js new file mode 100644 index 0000000..42c04b6 --- /dev/null +++ b/server-cache.js @@ -0,0 +1,115 @@ +const express = require('express'); +const app = express(); +const path = require('path'); +const bodyParser = require('body-parser'); + +const config = require('./config'); + +const log4js = require('log4js'); +const logger = log4js.getLogger(); +const URL = require('url'); +const http = require('http'); +const https = require('https'); + +const apicache = require('apicache'); + +logger.level = 'debug'; + +// app.use(compression()); +app.use(bodyParser.json()); +app.use(bodyParser.urlencoded({ 'extended': true })); + +const cache = apicache.middleware; + +app.use(cache('5 minutes')); + +function bob(req, res) { + logger.debug(req.params); + res.sendStatus(200); +} + +function getUrl (req, res) { + const theUrl = req.params.encoded_id; + logger.debug('Want', theUrl); + const options = URL.parse(theUrl); + options.followAllRedirects = true; + options.headers = { + 'User-Agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36' + }; + + // console.log('Options', options); + logger.info('>> getting url'); + function urlQuery (callback) { + try { + let count = 0; + http.request(options, responseHandler).end(); + + function responseHandler(response) { + console.log(`STATUS: ${response.statusCode}`); + console.log(`HEADERS: ${JSON.stringify(response.headers)}`); + response.setEncoding('utf8'); + if (response.statusCode === 302 || response.statusCode === 301) { + body = []; + const rUrl = URL.parse(response.headers.location); + + rUrl.followAllRedirects = true; + rUrl.headers = { + 'User-Agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36' + }; + + logger.debug('>> follow', response.headers.location, count); + // logger.debug(rUrl); + count++; + if (rUrl.protocol === 'https:') + https.request(rUrl, responseHandler).end(); + else + http.request(rUrl, responseHandler).end(); + + // this.request(response.headers.location); + // console.log(this); + } + + let data = ''; + response.on('data', chunk => { + data += chunk; + }); + response.on('end', () => { + if (response.statusCode !== 302 && response.statusCode !== 301) + callback(data); + }); + response.on('error', e => { + logger.error(e); + }); + } + + /* http.request(options).on('response', response => { + let data = ''; + response.on('data', chunk => { + data += chunk; + }); + response.on('end', () => { + callback(data); + }); + response.on('error', e => { + logger.error(e); + }); + }).end();*/ + } + catch (e) { + logger.error(e); + } + } + + urlQuery(a => { + // logger.info(a); + logger.info('Got result'); + // res.setHeader('Content-Type', 'application/json'); + res.send(a); + }); +} + +app.get('/:encoded_id', getUrl); + +const server = app.listen(config.port, () => { + logger.info(`Server listening on port ${config.port}`); +}); diff --git a/server.js b/server.js index 08743d3..818e50f 100644 --- a/server.js +++ b/server.js @@ -10,12 +10,16 @@ const logger = log4js.getLogger(); const URL = require('url'); const http = require('http'); const https = require('https'); +const apicache = require('apicache'); logger.level = 'debug'; // app.use(compression()); app.use(bodyParser.json()); app.use(bodyParser.urlencoded({ 'extended': true })); +const cache = apicache.middleware; + +app.use(cache('5 minutes')); function bob(req, res) { logger.debug(req.params);