added server caching

package-lock.json

@@ -39,7 +39,6 @@
       "version": "4.11.8",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-4.11.8.tgz",
       "integrity": "sha1-gv+wKynmYq5TvcIK8VlHcGc5xTY=",
-      "optional": true,
       "requires": {
         "co": "4.6.0",
         "json-stable-stringify": "1.0.1"
@@ -56,6 +55,11 @@
       "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
       "optional": true
     },
+    "apicache": {
+      "version": "0.11.2",
+      "resolved": "https://registry.npmjs.org/apicache/-/apicache-0.11.2.tgz",
+      "integrity": "sha512-olsFYz/fxxJPnZ3I8KG9CL40PaibfWhXJ+C0JO2ti4N3xsnCJ3plm50tj4eBguMqHaUBBVqsaFGyfZ8vpyZXEg=="
+    },
     "array-flatten": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
@@ -64,14 +68,12 @@
     "asn1": {
       "version": "0.2.3",
       "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz",
-      "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y=",
+      "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y="
-      "optional": true
     },
     "assert-plus": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz",
-      "integrity": "sha1-104bh+ev/A24qttwIfP+SBAasjQ=",
+      "integrity": "sha1-104bh+ev/A24qttwIfP+SBAasjQ="
-      "optional": true
     },
     "ast-types": {
       "version": "0.9.12",
@@ -91,20 +93,17 @@
     "asynckit": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
-      "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=",
+      "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
-      "optional": true
     },
     "aws-sign2": {
       "version": "0.6.0",
       "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz",
-      "integrity": "sha1-FDQt0428yU0OW4fXY81jYSwOeU8=",
+      "integrity": "sha1-FDQt0428yU0OW4fXY81jYSwOeU8="
-      "optional": true
     },
     "aws4": {
       "version": "1.6.0",
       "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.6.0.tgz",
-      "integrity": "sha1-g+9cqGCysy5KDe7e6MdxudtXRx4=",
+      "integrity": "sha1-g+9cqGCysy5KDe7e6MdxudtXRx4="
-      "optional": true
     },
     "axios": {
       "version": "0.15.3",
@@ -181,8 +180,7 @@
     "caseless": {
       "version": "0.12.0",
       "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
-      "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=",
+      "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
-      "optional": true
     },
     "chalk": {
       "version": "1.1.3",
@@ -200,8 +198,7 @@
     "co": {
       "version": "4.6.0",
       "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
-      "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=",
+      "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ="
-      "optional": true
     },
     "combined-stream": {
       "version": "1.0.5",
@@ -246,7 +243,6 @@
       "version": "2.0.5",
       "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz",
       "integrity": "sha1-O9/s3GCBR8HGcgL6KR59ylnqo7g=",
-      "optional": true,
       "requires": {
         "boom": "2.10.1"
       }
@@ -255,7 +251,6 @@
       "version": "1.14.1",
       "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
       "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=",
-      "optional": true,
       "requires": {
         "assert-plus": "1.0.0"
       },
@@ -263,8 +258,7 @@
         "assert-plus": {
           "version": "1.0.0",
           "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
-          "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
+          "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
-          "optional": true
         }
       }
     },
@@ -473,14 +467,12 @@
     "forever-agent": {
       "version": "0.6.1",
       "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
-      "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=",
+      "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE="
-      "optional": true
     },
     "form-data": {
       "version": "2.1.4",
       "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.1.4.tgz",
       "integrity": "sha1-M8GDrPGTJ27KqYFDpp6Uv+4XUNE=",
-      "optional": true,
       "requires": {
         "asynckit": "0.4.0",
         "combined-stream": "1.0.5",
@@ -560,7 +552,6 @@
       "version": "0.1.7",
       "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
       "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
-      "optional": true,
       "requires": {
         "assert-plus": "1.0.0"
       },
@@ -568,22 +559,19 @@
         "assert-plus": {
           "version": "1.0.0",
           "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
-          "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
+          "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
-          "optional": true
         }
       }
     },
     "har-schema": {
       "version": "1.0.5",
       "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-1.0.5.tgz",
-      "integrity": "sha1-0mMTX0MwfALGAq/I/pWXDAFRNp4=",
+      "integrity": "sha1-0mMTX0MwfALGAq/I/pWXDAFRNp4="
-      "optional": true
     },
     "har-validator": {
       "version": "4.2.1",
       "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-4.2.1.tgz",
       "integrity": "sha1-M0gdDxu/9gDdID11gSpqX7oALio=",
-      "optional": true,
       "requires": {
         "ajv": "4.11.8",
         "har-schema": "1.0.5"
@@ -602,7 +590,6 @@
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz",
       "integrity": "sha1-B4REvXwWQLD+VA0sm3PVlnjo4cQ=",
-      "optional": true,
       "requires": {
         "boom": "2.10.1",
         "cryptiles": "2.0.5",
@@ -655,7 +642,6 @@
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz",
       "integrity": "sha1-33LiZwZs0Kxn+3at+OE0qPvPkb8=",
-      "optional": true,
       "requires": {
         "assert-plus": "0.2.0",
         "jsprim": "1.4.1",
@@ -745,8 +731,7 @@
     "is-typedarray": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
-      "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=",
+      "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo="
-      "optional": true
     },
     "isarray": {
       "version": "1.0.0",
@@ -756,8 +741,7 @@
     "isstream": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
-      "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=",
+      "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
-      "optional": true
     },
     "jsbn": {
       "version": "0.1.1",
@@ -768,14 +752,12 @@
     "json-schema": {
       "version": "0.2.3",
       "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
-      "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=",
+      "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM="
-      "optional": true
     },
     "json-stable-stringify": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz",
       "integrity": "sha1-mnWdOcXy/1A/1TAGRu1EX4jE+a8=",
-      "optional": true,
       "requires": {
         "jsonify": "0.0.0"
       }
@@ -788,8 +770,7 @@
     "jsonify": {
       "version": "0.0.0",
       "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz",
-      "integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM=",
+      "integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM="
-      "optional": true
     },
     "jsonpointer": {
       "version": "4.0.1",
@@ -801,7 +782,6 @@
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
       "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=",
-      "optional": true,
       "requires": {
         "assert-plus": "1.0.0",
         "extsprintf": "1.3.0",
@@ -812,8 +792,7 @@
         "assert-plus": {
           "version": "1.0.0",
           "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
-          "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
+          "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
-          "optional": true
         }
       }
     },
@@ -1068,6 +1047,14 @@
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
       "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
     },
+    "nano-cache": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/nano-cache/-/nano-cache-1.1.1.tgz",
+      "integrity": "sha1-UQbZXzkstENZZU4hzAFa4hjnVGY=",
+      "requires": {
+        "extend": "3.0.1"
+      }
+    },
     "negotiator": {
       "version": "0.6.1",
       "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz",
@@ -1165,8 +1152,7 @@
     "oauth-sign": {
       "version": "0.8.2",
       "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz",
-      "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=",
+      "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM="
-      "optional": true
     },
     "on-finished": {
       "version": "2.3.0",
@@ -1267,8 +1253,7 @@
     "performance-now": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-0.2.0.tgz",
-      "integrity": "sha1-M+8wxcd9TqIcWlOGnZG1bY8lVeU=",
+      "integrity": "sha1-M+8wxcd9TqIcWlOGnZG1bY8lVeU="
-      "optional": true
     },
     "pinkie": {
       "version": "2.0.4",
@@ -1328,8 +1313,7 @@
     "punycode": {
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
-      "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=",
+      "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
-      "optional": true
     },
     "q": {
       "version": "1.4.1",
@@ -1404,7 +1388,6 @@
       "version": "2.81.0",
       "resolved": "https://registry.npmjs.org/request/-/request-2.81.0.tgz",
       "integrity": "sha1-xpKJRqDgbF+Nb4qTM0af/aRimKA=",
-      "optional": true,
       "requires": {
         "aws-sign2": "0.6.0",
         "aws4": "1.6.0",
@@ -1433,11 +1416,19 @@
         "qs": {
           "version": "6.4.0",
           "resolved": "https://registry.npmjs.org/qs/-/qs-6.4.0.tgz",
-          "integrity": "sha1-E+JtKK1rD/qpExLNO/cI7TUecjM=",
+          "integrity": "sha1-E+JtKK1rD/qpExLNO/cI7TUecjM="
-          "optional": true
         }
       }
     },
+    "request-promise-cache": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/request-promise-cache/-/request-promise-cache-1.0.6.tgz",
+      "integrity": "sha1-d8opARFlG0UnLzFA6cDoNXCHiMM=",
+      "requires": {
+        "nano-cache": "1.1.1",
+        "request": "2.81.0"
+      }
+    },
     "requestretry": {
       "version": "1.12.2",
       "resolved": "https://registry.npmjs.org/requestretry/-/requestretry-1.12.2.tgz",
@@ -1523,7 +1514,6 @@
       "version": "1.0.9",
       "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz",
       "integrity": "sha1-ZUEYTMkK7qbG57NeJlkIJEPGYZg=",
-      "optional": true,
       "requires": {
         "hoek": "2.16.3"
       }
@@ -1564,7 +1554,6 @@
       "version": "1.13.1",
       "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.13.1.tgz",
       "integrity": "sha1-US322mKHFEMW3EwY/hzx2UBzm+M=",
-      "optional": true,
       "requires": {
         "asn1": "0.2.3",
         "assert-plus": "1.0.0",
@@ -1579,8 +1568,7 @@
         "assert-plus": {
           "version": "1.0.0",
           "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
-          "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
+          "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
-          "optional": true
         }
       }
     },
@@ -1632,8 +1620,7 @@
     "stringstream": {
       "version": "0.0.5",
       "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz",
-      "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg=",
+      "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg="
-      "optional": true
     },
     "strip-ansi": {
       "version": "3.0.1",
@@ -1666,7 +1653,6 @@
       "version": "2.3.2",
       "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.2.tgz",
       "integrity": "sha1-8IH3bkyFcg5sN6X6ztc3FQ2EByo=",
-      "optional": true,
       "requires": {
         "punycode": "1.4.1"
       }
@@ -1681,7 +1667,6 @@
       "version": "0.6.0",
       "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
       "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
-      "optional": true,
       "requires": {
         "safe-buffer": "5.1.1"
       }
@@ -1763,8 +1748,7 @@
     "uuid": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz",
-      "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==",
+      "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g=="
-      "optional": true
     },
     "vary": {
       "version": "1.1.1",
@@ -1775,7 +1759,6 @@
       "version": "1.10.0",
       "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
       "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=",
-      "optional": true,
       "requires": {
         "assert-plus": "1.0.0",
         "core-util-is": "1.0.2",
@@ -1785,8 +1768,7 @@
         "assert-plus": {
           "version": "1.0.0",
           "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
-          "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=",
+          "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
-          "optional": true
         }
       }
     },

package.json

@@ -9,12 +9,14 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "apicache": "^0.11.2",
     "body-parser": "^1.18.0",
     "express": "^4.15.4",
     "http": "0.0.0",
     "https": "^1.0.0",
     "log4js": "^2.3.3",
     "path": "^0.12.7",
+    "request-promise-cache": "^1.0.6",
     "url": "^0.11.0"
   }
 }

server-cache.js

@@ -0,0 +1,115 @@
+const express = require('express');
+const app = express();
+const path = require('path');
+const bodyParser = require('body-parser');
+
+const config = require('./config');
+
+const log4js = require('log4js');
+const logger = log4js.getLogger();
+const URL = require('url');
+const http = require('http');
+const https = require('https');
+
+const apicache = require('apicache');
+
+logger.level = 'debug';
+
+// app.use(compression());
+app.use(bodyParser.json());
+app.use(bodyParser.urlencoded({ 'extended': true }));
+
+const cache = apicache.middleware;
+
+app.use(cache('5 minutes'));
+
+function bob(req, res) {
+  logger.debug(req.params);
+  res.sendStatus(200);
+}
+
+function getUrl (req, res) {
+  const theUrl = req.params.encoded_id;
+  logger.debug('Want', theUrl);
+  const options = URL.parse(theUrl);
+  options.followAllRedirects = true;
+  options.headers = {
+    'User-Agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
+  };
+
+  //  console.log('Options', options);
+  logger.info('>> getting url');
+  function urlQuery (callback) {
+    try {
+      let count = 0;
+      http.request(options, responseHandler).end();
+
+      function responseHandler(response) {
+        console.log(`STATUS: ${response.statusCode}`);
+        console.log(`HEADERS: ${JSON.stringify(response.headers)}`);
+        response.setEncoding('utf8');
+        if (response.statusCode === 302 || response.statusCode === 301) {
+          body = [];
+          const rUrl = URL.parse(response.headers.location);
+
+          rUrl.followAllRedirects = true;
+          rUrl.headers = {
+            'User-Agent' : 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
+          };
+
+          logger.debug('>> follow', response.headers.location, count);
+          // logger.debug(rUrl);
+          count++;
+          if (rUrl.protocol === 'https:') 
+            https.request(rUrl, responseHandler).end();
+          else 
+            http.request(rUrl, responseHandler).end();
+          
+          // this.request(response.headers.location);
+          // console.log(this);
+        }
+
+        let data = '';
+        response.on('data', chunk => {
+          data += chunk;
+        });
+        response.on('end', () => {
+          if (response.statusCode !== 302 && response.statusCode !== 301) 
+            callback(data);
+        });
+        response.on('error', e => {
+          logger.error(e);
+        });
+      }
+
+      /* http.request(options).on('response', response => {
+        let data = '';
+        response.on('data', chunk => {
+          data += chunk;
+        });
+        response.on('end', () => {
+          callback(data);
+        });
+        response.on('error', e => {
+          logger.error(e);
+        });
+      }).end();*/
+    }
+    catch (e) {
+      logger.error(e);
+    }
+  }
+
+  urlQuery(a => {
+    // logger.info(a);
+    logger.info('Got result');
+    // res.setHeader('Content-Type', 'application/json');
+    res.send(a);
+  });
+}
+
+app.get('/:encoded_id', getUrl);
+
+const server = app.listen(config.port, () => {
+  logger.info(`Server listening on port ${config.port}`);
+});

server.js

@@ -10,12 +10,16 @@ const logger = log4js.getLogger();
 const URL = require('url');
 const http = require('http');
 const https = require('https');
+const apicache = require('apicache');
 
 logger.level = 'debug';
 
 // app.use(compression());
 app.use(bodyParser.json());
 app.use(bodyParser.urlencoded({ 'extended': true }));
+const cache = apicache.middleware;
+
+app.use(cache('5 minutes'));
 
 function bob(req, res) {
   logger.debug(req.params);