From 7263b5e3db518fe139009d3767ee2830d6a4230a Mon Sep 17 00:00:00 2001 From: Martin Donnelly Date: Mon, 8 Apr 2019 14:34:41 +0100 Subject: [PATCH] added proxymesh --- package-lock.json | 59 +++++++++++++++++++++++++++++++++++------------ server.js | 54 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 96 insertions(+), 17 deletions(-) diff --git a/package-lock.json b/package-lock.json index 17ea2bc..cf47ea0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,6 +23,7 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.0.tgz", "integrity": "sha512-c+R/U5X+2zz2+UCrCFv6odQzJdoqI+YecuhnAJLa1zYaMc13zPfwMwZrr91Pd1DYNo/yPRbiM4WVf9whgwFsIg==", + "optional": true, "requires": { "es6-promisify": "^5.0.0" } @@ -54,7 +55,8 @@ "ansi-regex": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "optional": true }, "ansi-styles": { "version": "2.2.1", @@ -197,6 +199,7 @@ "version": "2.10.1", "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz", "integrity": "sha1-OciRjO/1eZ+D+UkqhI9iWt0Mdm8=", + "optional": true, "requires": { "hoek": "2.x.x" } @@ -204,7 +207,8 @@ "buffer-more-ints": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-0.0.2.tgz", - "integrity": "sha1-JrOIXRD6E9t/wBquOquHAZngEkw=" + "integrity": "sha1-JrOIXRD6E9t/wBquOquHAZngEkw=", + "optional": true }, "buildmail": { "version": "4.0.1", @@ -389,12 +393,14 @@ "es6-promise": { "version": "4.2.4", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.4.tgz", - "integrity": "sha512-/NdNZVJg+uZgtm9eS3O6lrOLYmQag2DjdEXuPaHlZ6RuVqgqaVZfgYCepEIKsLqwdQArOPtC3XzRLqGGfT8KQQ==" + "integrity": "sha512-/NdNZVJg+uZgtm9eS3O6lrOLYmQag2DjdEXuPaHlZ6RuVqgqaVZfgYCepEIKsLqwdQArOPtC3XzRLqGGfT8KQQ==", + "optional": true }, "es6-promisify": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", + "optional": true, "requires": { "es6-promise": "^4.0.3" } @@ -426,7 +432,8 @@ "esprima": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/esprima/-/esprima-3.1.3.tgz", - "integrity": "sha1-/cpRzuYTOJXjyI1TXOSdv/YqRjM=" + "integrity": "sha1-/cpRzuYTOJXjyI1TXOSdv/YqRjM=", + "optional": true }, "estraverse": { "version": "4.2.0", @@ -792,7 +799,8 @@ "hoek": { "version": "2.16.3", "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz", - "integrity": "sha1-ILt0A9POo5jpHcRxCo/xuCdKJe0=" + "integrity": "sha1-ILt0A9POo5jpHcRxCo/xuCdKJe0=", + "optional": true }, "http": { "version": "0.0.0", @@ -814,6 +822,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-2.1.0.tgz", "integrity": "sha512-qwHbBLV7WviBl0rQsOzH6o5lwyOIvwp/BdFnvVxXORldu5TmjFfjzBcWUWS5kWAZhmv+JtiDhSuQCp4sBfbIgg==", + "optional": true, "requires": { "agent-base": "4", "debug": "3.1.0" @@ -823,6 +832,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "optional": true, "requires": { "ms": "2.0.0" } @@ -843,6 +853,7 @@ "version": "1.6.1", "resolved": "https://registry.npmjs.org/httpntlm/-/httpntlm-1.6.1.tgz", "integrity": "sha1-rQFScUOi6Hc8+uapb1hla7UqNLI=", + "optional": true, "requires": { "httpreq": ">=0.4.22", "underscore": "~1.7.0" @@ -851,7 +862,8 @@ "httpreq": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/httpreq/-/httpreq-0.4.24.tgz", - "integrity": "sha1-QzX/2CzZaWaKOUZckprGHWOTYn8=" + "integrity": "sha1-QzX/2CzZaWaKOUZckprGHWOTYn8=", + "optional": true }, "https": { "version": "1.0.0", @@ -862,6 +874,7 @@ "version": "2.2.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz", "integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==", + "optional": true, "requires": { "agent-base": "^4.1.0", "debug": "^3.1.0" @@ -871,6 +884,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "optional": true, "requires": { "ms": "2.0.0" } @@ -899,7 +913,8 @@ "ip": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", - "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=" + "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=", + "optional": true }, "ip6": { "version": "0.0.4", @@ -1009,12 +1024,14 @@ "libbase64": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/libbase64/-/libbase64-0.1.0.tgz", - "integrity": "sha1-YjUag5VjrF/1vSbxL2Dpgwu3UeY=" + "integrity": "sha1-YjUag5VjrF/1vSbxL2Dpgwu3UeY=", + "optional": true }, "libmime": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/libmime/-/libmime-3.0.0.tgz", "integrity": "sha1-UaGp50SOy9Ms2lRCFnW7IbwJPaY=", + "optional": true, "requires": { "iconv-lite": "0.4.15", "libbase64": "0.1.0", @@ -1024,14 +1041,16 @@ "iconv-lite": { "version": "0.4.15", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.15.tgz", - "integrity": "sha1-/iZaIYrGpXz+hUkn6dBMGYJe3es=" + "integrity": "sha1-/iZaIYrGpXz+hUkn6dBMGYJe3es=", + "optional": true } } }, "libqp": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/libqp/-/libqp-1.1.0.tgz", - "integrity": "sha1-9ebgatdLeU+1tbZpiL9yjvHe2+g=" + "integrity": "sha1-9ebgatdLeU+1tbZpiL9yjvHe2+g=", + "optional": true }, "lodash": { "version": "4.17.10", @@ -1339,12 +1358,14 @@ "nodemailer-fetch": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/nodemailer-fetch/-/nodemailer-fetch-1.6.0.tgz", - "integrity": "sha1-ecSQihwPXzdbc/6IjamCj23JY6Q=" + "integrity": "sha1-ecSQihwPXzdbc/6IjamCj23JY6Q=", + "optional": true }, "nodemailer-shared": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/nodemailer-shared/-/nodemailer-shared-1.1.0.tgz", "integrity": "sha1-z1mU4v0mjQD1zw+nZ6CBae2wfsA=", + "optional": true, "requires": { "nodemailer-fetch": "1.6.0" } @@ -1374,7 +1395,8 @@ "nodemailer-wellknown": { "version": "0.1.10", "resolved": "https://registry.npmjs.org/nodemailer-wellknown/-/nodemailer-wellknown-0.1.10.tgz", - "integrity": "sha1-WG24EB2zDLRDjrVGc3pBqtDPE9U=" + "integrity": "sha1-WG24EB2zDLRDjrVGc3pBqtDPE9U=", + "optional": true }, "oauth-sign": { "version": "0.8.2", @@ -1502,7 +1524,8 @@ "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", - "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=" + "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", + "optional": true }, "process": { "version": "0.11.10", @@ -1767,12 +1790,14 @@ "smart-buffer": { "version": "1.1.15", "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-1.1.15.tgz", - "integrity": "sha1-fxFLW2X6s+KjWqd1uxLw0cZJvxY=" + "integrity": "sha1-fxFLW2X6s+KjWqd1uxLw0cZJvxY=", + "optional": true }, "smtp-connection": { "version": "2.12.0", "resolved": "https://registry.npmjs.org/smtp-connection/-/smtp-connection-2.12.0.tgz", "integrity": "sha1-1275EnyyPCJZ7bHoNJwujV4tdME=", + "optional": true, "requires": { "httpntlm": "1.6.1", "nodemailer-shared": "1.1.0" @@ -1791,6 +1816,7 @@ "version": "1.1.10", "resolved": "https://registry.npmjs.org/socks/-/socks-1.1.10.tgz", "integrity": "sha1-W4t/x8jzQcU+0FbpKbe/Tei6e1o=", + "optional": true, "requires": { "ip": "^1.1.4", "smart-buffer": "^1.0.13" @@ -1800,6 +1826,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-3.0.1.tgz", "integrity": "sha512-ZwEDymm204mTzvdqyUqOdovVr2YRd2NYskrYrF2LXyZ9qDiMAoFESGK8CRphiO7rtbo2Y757k2Nia3x2hGtalA==", + "optional": true, "requires": { "agent-base": "^4.1.0", "socks": "^1.1.10" @@ -1956,6 +1983,7 @@ "version": "0.3.2", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", + "optional": true, "requires": { "prelude-ls": "~1.1.2" } @@ -1972,7 +2000,8 @@ "underscore": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz", - "integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk=" + "integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk=", + "optional": true }, "unpipe": { "version": "1.0.0", diff --git a/server.js b/server.js index 66af26f..694e01b 100644 --- a/server.js +++ b/server.js @@ -12,6 +12,8 @@ const http = require('http'); const https = require('https'); const apicache = require('apicache'); +const request = require('request'); + const ipfilter = require('express-ipfilter').IpFilter; const IpDeniedError = require('express-ipfilter').IpDeniedError; @@ -34,7 +36,7 @@ const ips = ['212.71.255.44', '82.35.75.161', '54.146.189.113', '52.90.194.191'] const complexUrls = new Map(); -app.use(ipfilter(ips, { 'mode': 'allow' })); +// app.use(ipfilter(ips, { 'mode': 'allow' })); app.use((err, req, res, _next) => { const thisErr = Object.assign({}, err); @@ -214,7 +216,55 @@ function getUrl (req, res) { }); } -app.get('/:encoded_id', cache('15 minutes'), getUrl); + +function getUrlV2(req, res) { + let final; + const firstUrl = req.params.encoded_id; + // let theUrl; + logger.debug('Want', firstUrl); + const theUrl = complexUrls.get(firstUrl) || firstUrl; + + if (theUrl === undefined || bouncer.indexOf(theUrl) !== -1 || theUrl === '') { + logger.warn(`You're not getting in ${theUrl}`); + res.status(400).send(''); + + return; + } + + if (theUrl !== firstUrl) + logger.info(`>>> Using ${theUrl} instead of ${firstUrl}`); + + const options = urlparser.parse(theUrl); + options.followAllRedirects = true; + options.headers = { + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36' + }; + + // console.log('Options', options); + logger.info(`>> getting ${theUrl}`); + + + request({ + url : theUrl, + proxy : 'http://us-wa.proxymesh.com:31280', + tunnel : true + }, (err, _res, body) => { + + if (!err) { + res.send(body); + } else { + + res.send(''); + } + + + + }); + + +} + +app.get('/:encoded_id', cache('15 minutes'), getUrlV2); process.on('uncaughtException', function (err) { console.error(err);