added proxymesh

package-lock.json

@@ -23,6 +23,7 @@
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.0.tgz",
       "integrity": "sha512-c+R/U5X+2zz2+UCrCFv6odQzJdoqI+YecuhnAJLa1zYaMc13zPfwMwZrr91Pd1DYNo/yPRbiM4WVf9whgwFsIg==",
+      "optional": true,
       "requires": {
         "es6-promisify": "^5.0.0"
       }
@@ -54,7 +55,8 @@
     "ansi-regex": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
-      "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
+      "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=",
+      "optional": true
     },
     "ansi-styles": {
       "version": "2.2.1",
@@ -197,6 +199,7 @@
       "version": "2.10.1",
       "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz",
       "integrity": "sha1-OciRjO/1eZ+D+UkqhI9iWt0Mdm8=",
+      "optional": true,
       "requires": {
         "hoek": "2.x.x"
       }
@@ -204,7 +207,8 @@
     "buffer-more-ints": {
       "version": "0.0.2",
       "resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-0.0.2.tgz",
-      "integrity": "sha1-JrOIXRD6E9t/wBquOquHAZngEkw="
+      "integrity": "sha1-JrOIXRD6E9t/wBquOquHAZngEkw=",
+      "optional": true
     },
     "buildmail": {
       "version": "4.0.1",
@@ -389,12 +393,14 @@
     "es6-promise": {
       "version": "4.2.4",
       "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.4.tgz",
-      "integrity": "sha512-/NdNZVJg+uZgtm9eS3O6lrOLYmQag2DjdEXuPaHlZ6RuVqgqaVZfgYCepEIKsLqwdQArOPtC3XzRLqGGfT8KQQ=="
+      "integrity": "sha512-/NdNZVJg+uZgtm9eS3O6lrOLYmQag2DjdEXuPaHlZ6RuVqgqaVZfgYCepEIKsLqwdQArOPtC3XzRLqGGfT8KQQ==",
+      "optional": true
     },
     "es6-promisify": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz",
       "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=",
+      "optional": true,
       "requires": {
         "es6-promise": "^4.0.3"
       }
@@ -426,7 +432,8 @@
     "esprima": {
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/esprima/-/esprima-3.1.3.tgz",
-      "integrity": "sha1-/cpRzuYTOJXjyI1TXOSdv/YqRjM="
+      "integrity": "sha1-/cpRzuYTOJXjyI1TXOSdv/YqRjM=",
+      "optional": true
     },
     "estraverse": {
       "version": "4.2.0",
@@ -792,7 +799,8 @@
     "hoek": {
       "version": "2.16.3",
       "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz",
-      "integrity": "sha1-ILt0A9POo5jpHcRxCo/xuCdKJe0="
+      "integrity": "sha1-ILt0A9POo5jpHcRxCo/xuCdKJe0=",
+      "optional": true
     },
     "http": {
       "version": "0.0.0",
@@ -814,6 +822,7 @@
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-2.1.0.tgz",
       "integrity": "sha512-qwHbBLV7WviBl0rQsOzH6o5lwyOIvwp/BdFnvVxXORldu5TmjFfjzBcWUWS5kWAZhmv+JtiDhSuQCp4sBfbIgg==",
+      "optional": true,
       "requires": {
         "agent-base": "4",
         "debug": "3.1.0"
@@ -823,6 +832,7 @@
           "version": "3.1.0",
           "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
           "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+          "optional": true,
           "requires": {
             "ms": "2.0.0"
           }
@@ -843,6 +853,7 @@
       "version": "1.6.1",
       "resolved": "https://registry.npmjs.org/httpntlm/-/httpntlm-1.6.1.tgz",
       "integrity": "sha1-rQFScUOi6Hc8+uapb1hla7UqNLI=",
+      "optional": true,
       "requires": {
         "httpreq": ">=0.4.22",
         "underscore": "~1.7.0"
@@ -851,7 +862,8 @@
     "httpreq": {
       "version": "0.4.24",
       "resolved": "https://registry.npmjs.org/httpreq/-/httpreq-0.4.24.tgz",
-      "integrity": "sha1-QzX/2CzZaWaKOUZckprGHWOTYn8="
+      "integrity": "sha1-QzX/2CzZaWaKOUZckprGHWOTYn8=",
+      "optional": true
     },
     "https": {
       "version": "1.0.0",
@@ -862,6 +874,7 @@
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz",
       "integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==",
+      "optional": true,
       "requires": {
         "agent-base": "^4.1.0",
         "debug": "^3.1.0"
@@ -871,6 +884,7 @@
           "version": "3.1.0",
           "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
           "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
+          "optional": true,
           "requires": {
             "ms": "2.0.0"
           }
@@ -899,7 +913,8 @@
     "ip": {
       "version": "1.1.5",
       "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz",
-      "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo="
+      "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=",
+      "optional": true
     },
     "ip6": {
       "version": "0.0.4",
@@ -1009,12 +1024,14 @@
     "libbase64": {
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/libbase64/-/libbase64-0.1.0.tgz",
-      "integrity": "sha1-YjUag5VjrF/1vSbxL2Dpgwu3UeY="
+      "integrity": "sha1-YjUag5VjrF/1vSbxL2Dpgwu3UeY=",
+      "optional": true
     },
     "libmime": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/libmime/-/libmime-3.0.0.tgz",
       "integrity": "sha1-UaGp50SOy9Ms2lRCFnW7IbwJPaY=",
+      "optional": true,
       "requires": {
         "iconv-lite": "0.4.15",
         "libbase64": "0.1.0",
@@ -1024,14 +1041,16 @@
         "iconv-lite": {
           "version": "0.4.15",
           "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.15.tgz",
-          "integrity": "sha1-/iZaIYrGpXz+hUkn6dBMGYJe3es="
+          "integrity": "sha1-/iZaIYrGpXz+hUkn6dBMGYJe3es=",
+          "optional": true
         }
       }
     },
     "libqp": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/libqp/-/libqp-1.1.0.tgz",
-      "integrity": "sha1-9ebgatdLeU+1tbZpiL9yjvHe2+g="
+      "integrity": "sha1-9ebgatdLeU+1tbZpiL9yjvHe2+g=",
+      "optional": true
     },
     "lodash": {
       "version": "4.17.10",
@@ -1339,12 +1358,14 @@
     "nodemailer-fetch": {
       "version": "1.6.0",
       "resolved": "https://registry.npmjs.org/nodemailer-fetch/-/nodemailer-fetch-1.6.0.tgz",
-      "integrity": "sha1-ecSQihwPXzdbc/6IjamCj23JY6Q="
+      "integrity": "sha1-ecSQihwPXzdbc/6IjamCj23JY6Q=",
+      "optional": true
     },
     "nodemailer-shared": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/nodemailer-shared/-/nodemailer-shared-1.1.0.tgz",
       "integrity": "sha1-z1mU4v0mjQD1zw+nZ6CBae2wfsA=",
+      "optional": true,
       "requires": {
         "nodemailer-fetch": "1.6.0"
       }
@@ -1374,7 +1395,8 @@
     "nodemailer-wellknown": {
       "version": "0.1.10",
       "resolved": "https://registry.npmjs.org/nodemailer-wellknown/-/nodemailer-wellknown-0.1.10.tgz",
-      "integrity": "sha1-WG24EB2zDLRDjrVGc3pBqtDPE9U="
+      "integrity": "sha1-WG24EB2zDLRDjrVGc3pBqtDPE9U=",
+      "optional": true
     },
     "oauth-sign": {
       "version": "0.8.2",
@@ -1502,7 +1524,8 @@
     "prelude-ls": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz",
-      "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ="
+      "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=",
+      "optional": true
     },
     "process": {
       "version": "0.11.10",
@@ -1767,12 +1790,14 @@
     "smart-buffer": {
       "version": "1.1.15",
       "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-1.1.15.tgz",
-      "integrity": "sha1-fxFLW2X6s+KjWqd1uxLw0cZJvxY="
+      "integrity": "sha1-fxFLW2X6s+KjWqd1uxLw0cZJvxY=",
+      "optional": true
     },
     "smtp-connection": {
       "version": "2.12.0",
       "resolved": "https://registry.npmjs.org/smtp-connection/-/smtp-connection-2.12.0.tgz",
       "integrity": "sha1-1275EnyyPCJZ7bHoNJwujV4tdME=",
+      "optional": true,
       "requires": {
         "httpntlm": "1.6.1",
         "nodemailer-shared": "1.1.0"
@@ -1791,6 +1816,7 @@
       "version": "1.1.10",
       "resolved": "https://registry.npmjs.org/socks/-/socks-1.1.10.tgz",
       "integrity": "sha1-W4t/x8jzQcU+0FbpKbe/Tei6e1o=",
+      "optional": true,
       "requires": {
         "ip": "^1.1.4",
         "smart-buffer": "^1.0.13"
@@ -1800,6 +1826,7 @@
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-3.0.1.tgz",
       "integrity": "sha512-ZwEDymm204mTzvdqyUqOdovVr2YRd2NYskrYrF2LXyZ9qDiMAoFESGK8CRphiO7rtbo2Y757k2Nia3x2hGtalA==",
+      "optional": true,
       "requires": {
         "agent-base": "^4.1.0",
         "socks": "^1.1.10"
@@ -1956,6 +1983,7 @@
       "version": "0.3.2",
       "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz",
       "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=",
+      "optional": true,
       "requires": {
         "prelude-ls": "~1.1.2"
       }
@@ -1972,7 +2000,8 @@
     "underscore": {
       "version": "1.7.0",
       "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz",
-      "integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk="
+      "integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk=",
+      "optional": true
     },
     "unpipe": {
       "version": "1.0.0",

server.js

@@ -12,6 +12,8 @@ const http = require('http');
 const https = require('https');
 const apicache = require('apicache');
 
+const request = require('request');
+
 const ipfilter = require('express-ipfilter').IpFilter;
 const IpDeniedError = require('express-ipfilter').IpDeniedError;
 
@@ -34,7 +36,7 @@ const ips = ['212.71.255.44', '82.35.75.161', '54.146.189.113', '52.90.194.191']
 
 const complexUrls = new Map();
 
-app.use(ipfilter(ips, { 'mode': 'allow' }));
+// app.use(ipfilter(ips, { 'mode': 'allow' }));
 
 app.use((err, req, res, _next) => {
   const thisErr = Object.assign({}, err);
@@ -214,7 +216,55 @@ function getUrl (req, res) {
   });
 }
 
-app.get('/:encoded_id', cache('15 minutes'), getUrl);
+
+function getUrlV2(req, res) {
+  let final;
+  const firstUrl = req.params.encoded_id;
+  // let theUrl;
+  logger.debug('Want', firstUrl);
+  const theUrl = complexUrls.get(firstUrl) || firstUrl;
+
+  if (theUrl === undefined || bouncer.indexOf(theUrl) !== -1 || theUrl === '') {
+    logger.warn(`You're not getting in ${theUrl}`);
+    res.status(400).send('');
+
+    return;
+  }
+
+  if (theUrl !== firstUrl)
+    logger.info(`>>> Using ${theUrl} instead of ${firstUrl}`);
+
+  const options = urlparser.parse(theUrl);
+  options.followAllRedirects = true;
+  options.headers = {
+    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
+  };
+
+  //  console.log('Options', options);
+  logger.info(`>> getting ${theUrl}`);
+
+
+  request({
+    url    : theUrl,
+    proxy  : 'http://us-wa.proxymesh.com:31280',
+    tunnel : true
+  }, (err, _res, body) => {
+
+    if (!err) {
+      res.send(body);
+    } else {
+
+      res.send('');
+    }
+
+
+
+  });
+
+
+}
+
+app.get('/:encoded_id', cache('15 minutes'), getUrlV2);
 
 process.on('uncaughtException', function (err) {
   console.error(err);