Merge branch 'feature/websockets' into development

* feature/websockets:
  * Twitter now cleans db on restart
  wip

.gitignore

@@ -151,3 +151,4 @@ fabric.properties
 /src/es2016/bundle.js.map
 /src/backbone/bundle.js.map
 
+!/db/twitter.db

ecosystem.config.js

@@ -0,0 +1,14 @@
+module.exports = {
+  'apps' : [{
+    'name': 'Traintimes Server',
+    'script': './server.js',
+    'watch': ['./server.js', 'live/build']
+  }, {
+    'name': 'Twitter Grabber',
+    'script': './twitter.js',
+    'watch': './twitter.js',
+    'watch_delay': 10000,
+    'cron_restart' : '0 4 * * */3'
+  }]
+  
+};

live/build/fonts/fujicons.css

@@ -275,3 +275,48 @@
 .fa-clockface-filled:before {
   content: "\E9AA"
 }
+
+.fa-twitter:before {
+  content: "\EAE5"
+}
+
+.fa-cog:before {
+  content: "\EA9E"
+}
+
+.fa-cog-solid:before {
+  content: "\EA11"
+}
+
+.fa-foursquare:before {
+  content: "\EA0B"
+}
+
+
+
+
+
+
+
+
+
+
+/*full list*/
+
+.fa-binoculars-solid:before {
+  content: "\E900"
+}
+
+.fa-binoculars:before {
+  content: "\E901"
+}
+
+.fa-check-circle-solid:before {
+  content: "\E902"
+}
+
+.fa-check-circle:before {
+  content: "\E903"
+}
+
+

notes.md

@@ -0,0 +1,6 @@
+# notes
+
+
+```
+git flow feature finish websockets
+```

old_live/browserconfig.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<browserconfig>
+    <msapplication>
+        <tile>
+            <square150x150logo src="/img/mstile-150x150.png"/>
+            <TileColor>#2b5797</TileColor>
+        </tile>
+    </msapplication>
+</browserconfig>

old_live/fonts/fonts.css

@@ -0,0 +1,32 @@
+@font-face {
+	font-family: 'Roboto';
+	font-style: normal;
+	font-weight: 400;
+	src: url(Roboto-normal-400.woff) format('woff');
+	unicode-range: U+0-10FFFF;
+}
+
+@font-face {
+	font-family: 'Roboto Condensed';
+	font-style: normal;
+	font-weight: 400;
+	src: url(Roboto_Condensed-normal-400.woff) format('woff');
+	unicode-range: U+0-10FFFF;
+}
+
+@font-face {
+	font-family: 'Roboto Mono';
+	font-style: normal;
+	font-weight: 400;
+	src: url(Roboto_Mono-normal-400.woff) format('woff');
+	unicode-range: U+0-10FFFF;
+}
+
+@font-face {
+	font-family: 'Roboto Slab';
+	font-style: normal;
+	font-weight: 400;
+	src: url(Roboto_Slab-normal-400.woff) format('woff');
+	unicode-range: U+0-10FFFF;
+}
+

old_live/fonts/fujicons.css

@@ -0,0 +1,209 @@
+@font-face {
+    font-family: 'Fujicons';
+    font-style: normal;
+    font-weight: 400;
+    src: url(fujicons.ttf) format('truetype');
+    unicode-range: U+0-10FFFF;
+}
+
+
+.fa {
+  display: inline-block;
+  font: normal normal normal 14px/1 Fujicons;
+  font-size: inherit;
+  text-rendering: auto;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+}
+/* makes the font 33% larger relative to the icon container */
+.fa-lg {
+  font-size: 1.33333333em;
+  line-height: 0.75em;
+  vertical-align: -15%;
+}
+.fa-2x {
+  font-size: 2em;
+}
+.fa-3x {
+  font-size: 3em;
+}
+.fa-4x {
+  font-size: 4em;
+}
+.fa-5x {
+  font-size: 5em;
+}
+.fa-fw {
+  width: 1.28571429em;
+  text-align: center;
+}
+.fa-ul {
+  padding-left: 0;
+  margin-left: 2.14285714em;
+  list-style-type: none;
+}
+.fa-ul > li {
+  position: relative;
+}
+.fa-li {
+  position: absolute;
+  left: -2.14285714em;
+  width: 2.14285714em;
+  top: 0.14285714em;
+  text-align: center;
+}
+.fa-li.fa-lg {
+  left: -1.85714286em;
+}
+.fa-border {
+  padding: .2em .25em .15em;
+  border: solid 0.08em #eeeeee;
+  border-radius: .1em;
+}
+.fa-pull-left {
+  float: left;
+}
+.fa-pull-right {
+  float: right;
+}
+.fa.fa-pull-left {
+  margin-right: .3em;
+}
+.fa.fa-pull-right {
+  margin-left: .3em;
+}
+/* Deprecated as of 4.4.0 */
+.pull-right {
+  float: right;
+}
+.pull-left {
+  float: left;
+}
+.fa.pull-left {
+  margin-right: .3em;
+}
+.fa.pull-right {
+  margin-left: .3em;
+}
+.fa-spin {
+  -webkit-animation: fa-spin 2s infinite linear;
+  animation: fa-spin 2s infinite linear;
+}
+.fa-pulse {
+  -webkit-animation: fa-spin 1s infinite steps(8);
+  animation: fa-spin 1s infinite steps(8);
+}
+@-webkit-keyframes fa-spin {
+  0% {
+    -webkit-transform: rotate(0deg);
+    transform: rotate(0deg);
+  }
+  100% {
+    -webkit-transform: rotate(359deg);
+    transform: rotate(359deg);
+  }
+}
+@keyframes fa-spin {
+  0% {
+    -webkit-transform: rotate(0deg);
+    transform: rotate(0deg);
+  }
+  100% {
+    -webkit-transform: rotate(359deg);
+    transform: rotate(359deg);
+  }
+}
+.fa-rotate-90 {
+  -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=1)";
+  -webkit-transform: rotate(90deg);
+  -ms-transform: rotate(90deg);
+  transform: rotate(90deg);
+}
+.fa-rotate-180 {
+  -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=2)";
+  -webkit-transform: rotate(180deg);
+  -ms-transform: rotate(180deg);
+  transform: rotate(180deg);
+}
+.fa-rotate-270 {
+  -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=3)";
+  -webkit-transform: rotate(270deg);
+  -ms-transform: rotate(270deg);
+  transform: rotate(270deg);
+}
+.fa-flip-horizontal {
+  -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1)";
+  -webkit-transform: scale(-1, 1);
+  -ms-transform: scale(-1, 1);
+  transform: scale(-1, 1);
+}
+.fa-flip-vertical {
+  -ms-filter: "progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1)";
+  -webkit-transform: scale(1, -1);
+  -ms-transform: scale(1, -1);
+  transform: scale(1, -1);
+}
+:root .fa-rotate-90,
+:root .fa-rotate-180,
+:root .fa-rotate-270,
+:root .fa-flip-horizontal,
+:root .fa-flip-vertical {
+  filter: none;
+}
+.fa-stack {
+  position: relative;
+  display: inline-block;
+  width: 2em;
+  height: 2em;
+  line-height: 2em;
+  vertical-align: middle;
+}
+.fa-stack-1x,
+.fa-stack-2x {
+  position: absolute;
+  left: 0;
+  width: 100%;
+  text-align: center;
+}
+.fa-stack-1x {
+  line-height: inherit;
+}
+.fa-stack-2x {
+  font-size: 2em;
+}
+.fa-inverse {
+  color: #ffffff;
+}
+
+
+.fa-back:before {
+  content: "";
+}
+
+.fa-forward:before {
+  content: "";
+}
+
+.fa-globe:before {
+  content: "\EA12"
+}
+
+.fa-up:before {
+  content: "\E925"
+}
+
+.fa-down:before {
+  content: "\E922"
+}
+
+.fa-work:before {
+  content: "\E998"
+}
+
+.fa-home:before {
+  content: "\EA1E"
+}
+
+.fa-refresh:before {
+  content: "\EA88"
+}

old_live/fonts/gotham.css

@@ -0,0 +1,39 @@
+@font-face {
+    font-family: 'Gotham';
+    font-style: normal;
+    font-weight: 400;
+    src: url(GothamSSm-Book.otf) format('opentype');
+    unicode-range: U+0-10FFFF;
+}
+
+@font-face {
+    font-family: 'Gotham Light';
+    font-style: normal;
+    font-weight: 400;
+    src: url(GothamSSm-Light.otf) format('opentype');
+    unicode-range: U+0-10FFFF;
+}
+
+@font-face {
+    font-family: 'Gotham Medium';
+    font-style: normal;
+    font-weight: 400;
+    src: url(GothamSSm-Medium.otf) format('opentype');
+    unicode-range: U+0-10FFFF;
+}
+
+@font-face {
+    font-family: 'Gotham Bold';
+    font-style: normal;
+    font-weight: 400;
+    src: url(GothamSSm-Bold.otf) format('opentype');
+    unicode-range: U+0-10FFFF;
+}
+
+@font-face {
+    font-family: 'Gotham Black';
+    font-style: normal;
+    font-weight: 400;
+    src: url(GothamSSm-Black.otf) format('opentype');
+    unicode-range: U+0-10FFFF;
+}

old_live/img/safari-pinned-tab.svg

@@ -0,0 +1,33 @@
+<?xml version="1.0" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 20010904//EN"
+ "http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/svg10.dtd">
+<svg version="1.0" xmlns="http://www.w3.org/2000/svg"
+ width="192.000000pt" height="192.000000pt" viewBox="0 0 192.000000 192.000000"
+ preserveAspectRatio="xMidYMid meet">
+<metadata>
+Created by potrace 1.11, written by Peter Selinger 2001-2013
+</metadata>
+<g transform="translate(0.000000,192.000000) scale(0.100000,-0.100000)"
+fill="#000000" stroke="none">
+<path d="M440 1789 c-80 -4 -104 -10 -149 -35 -63 -34 -117 -94 -145 -162 -19
+-45 -20 -74 -20 -637 l0 -590 25 -56 c32 -67 91 -128 156 -160 l48 -23 605 -1
+605 0 53 28 c69 35 118 86 151 156 l26 56 0 585 c-1 534 -2 590 -18 637 -33
+94 -133 180 -231 199 -47 8 -969 11 -1106 3z m389 -79 c21 0 51 -44 51 -74 l0
+-34 80 -1 c79 -1 80 -1 80 23 0 40 21 74 51 85 57 20 112 -21 112 -83 0 -17 5
+-26 16 -26 68 0 143 -50 172 -115 17 -37 19 -74 19 -455 0 -265 -4 -429 -11
+-454 -6 -23 -28 -55 -55 -81 l-44 -43 85 -89 c91 -94 101 -118 63 -153 -41
+-38 -61 -28 -194 105 l-125 125 -168 0 -167 0 -128 -125 c-136 -134 -157 -145
+-196 -103 -35 38 -26 60 65 155 l85 88 -35 30 c-19 16 -44 48 -55 70 -18 38
+-19 65 -19 468 -1 404 1 431 19 467 32 63 105 110 171 110 11 0 16 8 14 23 -3
+48 53 104 93 91 8 -2 17 -4 21 -4z"/>
+<path d="M819 1532 c-48 -15 -65 -75 -31 -106 16 -14 43 -16 172 -17 169 0
+190 7 190 62 0 25 -21 57 -38 60 -22 4 -282 4 -293 1z"/>
+<path d="M735 1342 c-93 -5 -94 -6 -94 -192 0 -143 2 -160 19 -175 18 -16 48
+-18 301 -18 l282 -1 18 23 c17 20 19 42 19 176 0 133 -2 154 -18 168 -15 14
+-52 17 -242 19 -124 2 -252 2 -285 0z"/>
+<path d="M685 741 c-35 -22 -50 -67 -35 -107 12 -32 55 -57 93 -56 44 1 88 50
+81 89 -9 52 -15 63 -43 78 -34 17 -63 16 -96 -4z"/>
+<path d="M1138 745 c-49 -27 -58 -97 -18 -137 55 -54 139 -31 154 43 7 38 -9
+71 -44 92 -34 21 -56 21 -92 2z"/>
+</g>
+</svg>

old_live/index.html

@@ -0,0 +1,75 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Train Times</title>
+    <link href="fonts/fonts.css" rel="stylesheet">
+    <link href="fonts/gotham.css" rel="stylesheet">
+    <link href="fonts/fujicons.css" rel="stylesheet" type="text/css"/>
+    <link href="css/mui.custom.css" rel="stylesheet" type="text/css"/>
+
+    <link rel="apple-touch-icon" sizes="180x180" href="/img/apple-touch-icon.png">
+    <link rel="icon" type="image/png" sizes="32x32" href="/img/favicon-32x32.png">
+    <link rel="icon" type="image/png" sizes="16x16" href="/img/favicon-16x16.png">
+    <link rel="manifest" href="/manifest.json">
+    <link rel="mask-icon" href="/img/safari-pinned-tab.svg" color="#5bbad5">
+    <meta name="apple-mobile-web-app-title" content="Train Times">
+    <meta name="application-name" content="Train Times">
+    <meta name="theme-color" content="#ffffff">
+
+    <link href="css/style.min.css" rel="stylesheet" type="text/css"/>
+
+</head>
+<body>
+<div id="_hero" style="
+    position: absolute;
+    top: 0;
+    left: 0;
+    width: 100%;
+    height: 100%;
+"></div>
+<div class="appPanel" data-id="main">
+    <header id="header">
+        <div class="mui-appbar mui--appbar-line-height mui--z2">
+            <div class='mui-col-xs-8 mui-col-md-8 mui--appbar-height titleBar'>Train Times</div>
+            <div class='mui-col-xs-4 mui-col-md-4 mui--appbar-height'></div>
+        </div>
+    </header>
+
+    <div class="mui--appbar-height"></div>
+    <div class="mui-container">
+        <div class="app"></div>
+        <div id="trains" ></div>
+
+        <!--<div id='trainResults' class="mui&#45;&#45;hide"></div>-->
+    </div>
+
+
+</div>
+
+
+<div id="snackbar">⚡ Offline, waiting to reconnect...</div>
+
+
+<script type="text/template" id="modal-template">
+    <!--<div class="bbm-modal__topbar">
+      <h3 class="bbm-modal__title">Backbone.Modal</h3>
+    </div>-->
+    <div class="bbm-modal__section" id="modalContent">
+
+    </div>
+    <div class="bbm-modal__bottombar">
+        <a href="#" class="bbm-button">close</a>
+    </div>
+</script>
+
+
+<!--<script src="//cdn.muicss.com/mui-0.9.26/js/mui.min.js"></script>-->
+<script src="js/vendor.js" async></script>
+<script src="js/bundle.js" async></script>
+</body>
+
+</html>
+

old_live/manifest.json

@@ -0,0 +1,46 @@
+{
+    "name": "Train Times",
+    "short_name": "Train Times",
+    "icons": [
+        {
+            "src": "/img/Icon-36.png",
+            "sizes": "36x36",
+            "type": "image/png"
+        },
+        {
+            "src": "/img/Icon-48.png",
+            "sizes": "48x48",
+            "type": "image/png"
+        },
+        {
+            "src": "/img/Icon-72.png",
+            "sizes": "72x72",
+            "type": "image/png"
+        },
+        {
+            "src": "/img/Icon-96.png",
+            "sizes": "96x96",
+            "type": "image/png"
+        },
+        {
+            "src": "/img/Icon-144.png",
+            "sizes": "144x144",
+            "type": "image/png"
+        },
+        {
+            "src": "/img/Icon-192.png",
+            "sizes": "192x192",
+            "type": "image/png"
+        },
+        {
+            "src": "/img/Icon-512.png",
+            "sizes": "512x512",
+            "type": "image/png"
+        }
+    ],
+    "theme_color": "#ffffff",
+    "background_color": "#ffffff",
+    "start_url": ".",
+    "imgdisplay": "standalone",
+    "display": "standalone"
+}

old_live/service-worker.js

@@ -0,0 +1,113 @@
+// Copyright 2016 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+const CACHE_VERSION = 7;
+const dataCacheName = `traintimesData-v${CACHE_VERSION}`;
+const cacheName = `traintimePWA-final-${CACHE_VERSION}`;
+const filesToCache = [
+  '/',
+  '/index.html',
+  '/service-worker.js',
+  '/manifest.json',
+  '/browserconfig.xml',
+  '/css/style.min.css',
+  '/css/mui.custom.css',
+  '/fonts/fonts.css',
+  '/fonts/gotham.css',
+  '/fonts/Roboto-normal-400.woff',
+  '/fonts/Roboto_Mono-normal-400.woff',
+  '/js/bundle.js',
+  '/js/vendor.js',
+  '/img/Icon-36.png',
+  '/img/Icon-48.png',
+  '/img/Icon-72.png',
+  '/img/Icon-96.png',
+  '/img/Icon-144.png',
+  '/img/Icon-192.png',
+  '/img/Icon-512.png'
+];
+
+self.addEventListener('install', function(e) {
+  console.log('[ServiceWorker] Install');
+  e.waitUntil(
+    caches.open(cacheName).then(function(cache) {
+      console.log('[ServiceWorker] Caching app shell');
+
+      return cache.addAll(filesToCache);
+    })
+  );
+});
+
+self.addEventListener('activate', function(e) {
+  console.log('[ServiceWorker] Activate');
+  e.waitUntil(
+    caches.keys().then(function(keyList) {
+      return Promise.all(keyList.map(function(key) {
+        if (key !== cacheName && key !== dataCacheName) {
+          console.log('[ServiceWorker] Removing old cache', key);
+
+          return caches.delete(key);
+        }
+      }));
+    })
+  );
+
+  /*
+   * Fixes a corner case in which the app wasn't returning the latest data.
+   * You can reproduce the corner case by commenting out the line below and
+   * then doing the following steps: 1) load app for first time so that the
+   * initial New York City data is shown 2) press the refresh button on the
+   * app 3) go offline 4) reload the app. You expect to see the newer NYC
+   * data, but you actually see the initial data. This happens because the
+   * service worker is not yet activated. The code below essentially lets
+   * you activate the service worker faster.
+   */
+  return self.clients.claim();
+});
+
+self.addEventListener('fetch', function(e) {
+  console.warn('[Service Worker] Fetch', e.request.url);
+  const dataUrl = '/getnexttraintimes?';
+  if (e.request.url.indexOf(dataUrl) > -1) {
+    console.log('!');
+
+    /*
+     * When the request URL contains dataUrl, the app is asking for fresh
+     * weather data. In this case, the service worker always goes to the
+     * network and then caches the response. This is called the "Cache then
+     * network" strategy:
+     * https://jakearchibald.com/2014/offline-cookbook/#cache-then-network
+     */
+    e.respondWith(
+      caches.open(dataCacheName).then(function(cache) {
+        return fetch(e.request).then(function(response) {
+          cache.put(e.request.url, response.clone());
+
+          return response;
+        });
+      })
+    );
+  }
+  else
+
+    /*
+     * The app is asking for app shell files. In this scenario the app uses the
+     * "Cache, falling back to the network" offline strategy:
+     * https://jakearchibald.com/2014/offline-cookbook/#cache-falling-back-to-network
+     */
+    e.respondWith(
+      caches.match(e.request).then(function(response) {
+        return response || fetch(e.request);
+      })
+    );
+});

package-lock.json

@@ -1195,6 +1195,14 @@
       "integrity": "sha1-RqoXUftqL5PuXmibsQh9SxTGwgU=",
       "dev": true
     },
+    "bindings": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
+      "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
+      "requires": {
+        "file-uri-to-path": "1.0.0"
+      }
+    },
     "bl": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/bl/-/bl-1.1.2.tgz",
@@ -1915,6 +1923,11 @@
       "integrity": "sha1-lGfQMsOM+u+58teVASUwYvh/ob0=",
       "dev": true
     },
+    "cuid": {
+      "version": "2.1.8",
+      "resolved": "https://registry.npmjs.org/cuid/-/cuid-2.1.8.tgz",
+      "integrity": "sha512-xiEMER6E7TlTPnDxrM4eRiC6TRgjNX9xzEZ5U/Se2YJKr7Mq4pJn/2XEHjl3STcSh96GmkHPcBXLES8M29wyyg=="
+    },
     "dashdash": {
       "version": "1.14.1",
       "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
@@ -2162,6 +2175,11 @@
       "integrity": "sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw=",
       "optional": true
     },
+    "duplexer": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz",
+      "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg=="
+    },
     "duplexer2": {
       "version": "0.1.4",
       "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz",
@@ -2570,6 +2588,20 @@
       "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
       "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc="
     },
+    "event-stream": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-4.0.1.tgz",
+      "integrity": "sha512-qACXdu/9VHPBzcyhdOWR5/IahhGMf0roTeZJfzz077GwylcDd90yOHLouhmv7GJ5XzPi6ekaQWd8AvPP2nOvpA==",
+      "requires": {
+        "duplexer": "^0.1.1",
+        "from": "^0.1.7",
+        "map-stream": "0.0.7",
+        "pause-stream": "^0.0.11",
+        "split": "^1.0.1",
+        "stream-combiner": "^0.2.2",
+        "through": "^2.3.8"
+      }
+    },
     "events": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz",
@@ -2740,8 +2772,7 @@
     "file-uri-to-path": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
-      "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==",
-      "optional": true
+      "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw=="
     },
     "filename-regex": {
       "version": "2.0.1",
@@ -2926,6 +2957,11 @@
       "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
       "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac="
     },
+    "from": {
+      "version": "0.1.7",
+      "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz",
+      "integrity": "sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4="
+    },
     "fs-minipass": {
       "version": "1.2.7",
       "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz",
@@ -5114,8 +5150,7 @@
     "lodash.assign": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz",
-      "integrity": "sha1-DZnzzNem0mHRm9rrkkUAXShYCOc=",
-      "dev": true
+      "integrity": "sha1-DZnzzNem0mHRm9rrkkUAXShYCOc="
     },
     "lodash.create": {
       "version": "3.1.1",
@@ -5437,6 +5472,11 @@
         "pify": "^3.0.0"
       }
     },
+    "map-stream": {
+      "version": "0.0.7",
+      "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz",
+      "integrity": "sha1-ih8HiW2CsQkmvTdEokIACfiJdKg="
+    },
     "md5.js": {
       "version": "1.3.4",
       "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.4.tgz",
@@ -5526,6 +5566,11 @@
       "integrity": "sha1-5md4PZLonb00KBi1IwudYqZyrRg=",
       "dev": true
     },
+    "mingo": {
+      "version": "1.3.3",
+      "resolved": "https://registry.npmjs.org/mingo/-/mingo-1.3.3.tgz",
+      "integrity": "sha1-aSLE0Ufvx3GgFCWixMj3eER4xUY="
+    },
     "minibus": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/minibus/-/minibus-3.1.0.tgz",
@@ -5793,6 +5838,14 @@
         }
       }
     },
+    "node-crc": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/node-crc/-/node-crc-1.3.0.tgz",
+      "integrity": "sha512-XSs9gZhZKdiwpJDLSoRQsnn8N9q/KH4bc0ayO6+qnHbtb1YfIrdKfjbOQFCwup4q/D2sNxhbupvrZ3rWmzAk4A==",
+      "requires": {
+        "bindings": "^1.3.0"
+      }
+    },
     "node-fetch": {
       "version": "1.7.3",
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz",
@@ -6388,6 +6441,14 @@
       "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
       "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
     },
+    "pause-stream": {
+      "version": "0.0.11",
+      "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz",
+      "integrity": "sha1-/lo0sMvOErWqaitAPuLnO2AvFEU=",
+      "requires": {
+        "through": "~2.3"
+      }
+    },
     "pbkdf2": {
       "version": "3.0.14",
       "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.0.14.tgz",
@@ -7038,6 +7099,32 @@
       "integrity": "sha512-1HwIYD/8UlOtFS3QO3w7ey+SdSDFE4HRNLZoZRYVQefrOY3l17epswImeB1ijgJFQJodIaHcwkp3r/myBjFVbg==",
       "dev": true
     },
+    "save": {
+      "version": "2.4.0",
+      "resolved": "https://registry.npmjs.org/save/-/save-2.4.0.tgz",
+      "integrity": "sha512-wd5L2uVnsKYkIUaK6i8Ie66IOHaI328gMF0MPuTJtYOjXgUolC33LSIk7Qr8WVA55QHaGwfiVS8a7EFIeGOR3w==",
+      "requires": {
+        "async": "^2.6.2",
+        "event-stream": "^4.0.1",
+        "lodash.assign": "^4.2.0",
+        "mingo": "1"
+      },
+      "dependencies": {
+        "async": {
+          "version": "2.6.3",
+          "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz",
+          "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==",
+          "requires": {
+            "lodash": "^4.17.14"
+          }
+        },
+        "lodash": {
+          "version": "4.17.20",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
+          "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA=="
+        }
+      }
+    },
     "sax": {
       "version": "1.2.4",
       "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
@@ -7348,6 +7435,14 @@
         "source-map": "^0.5.6"
       }
     },
+    "split": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz",
+      "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==",
+      "requires": {
+        "through": "2"
+      }
+    },
     "sprintf-js": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
@@ -7400,6 +7495,15 @@
         "readable-stream": "^2.0.2"
       }
     },
+    "stream-combiner": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz",
+      "integrity": "sha1-rsjLrBd7Vrb0+kec7YwZEs7lKFg=",
+      "requires": {
+        "duplexer": "~0.1.1",
+        "through": "~2.3.4"
+      }
+    },
     "stream-combiner2": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/stream-combiner2/-/stream-combiner2-1.1.1.tgz",
@@ -8775,6 +8879,11 @@
         "mkdirp": "^0.5.1"
       }
     },
+    "ws": {
+      "version": "7.3.1",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-7.3.1.tgz",
+      "integrity": "sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA=="
+    },
     "xregexp": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-2.0.0.tgz",

package.json

@@ -3,9 +3,7 @@
   "version": "1.0.0",
   "description": "Train Times Progressive Web App",
   "main": "app.js",
-  "scripts": {
-
-  },
+  "scripts": {},
   "author": "",
   "license": "ISC",
   "dependencies": {
@@ -15,6 +13,7 @@
     "browserify": "^14.5.0",
     "compressjs": "^1.0.3",
     "cors": "^2.8.4",
+    "cuid": "^2.1.8",
     "dotenv": "^8.2.0",
     "es6-promise": "^4.1.1",
     "express": "^4.16.2",
@@ -25,9 +24,12 @@
     "lz77": "^1.1.0",
     "minibus": "^3.1.0",
     "muicss": "^0.9.33",
+    "node-crc": "^1.3.0",
+    "save": "^2.4.0",
     "sqlite3": "^4.2.0",
     "twitter-lite": "^0.11.0",
-    "underscore": "^1.8.3"
+    "underscore": "^1.8.3",
+    "ws": "^7.3.1"
   },
   "devDependencies": {
     "babel-cli": "^6.26.0",

server.js

@@ -1,3 +1,4 @@
+const http = require('http');
 const express = require('express');
 const cors = require('cors');
 
@@ -7,6 +8,13 @@ const logger = require('log4js').getLogger('Server');
 
 const train = require('./server/lib/train');
 const twitter = require('./server/lib/twitter');
+
+const WebSocket = require('ws');
+const SocketHandler = require('./server/lib/wshandlerv3');
+
+const Events = require('events');
+const busEmitter = new Events.EventEmitter();
+
 logger.level = 'debug';
 
 const app = express();
@@ -34,9 +42,29 @@ process.on('uncaughtException', err => {
   logger.error(err);
 });
 
-app.listen(port, (err) => {
+/* app.listen(port, (err) => {
   if (err)
     return logger.error('Server error:', err);
 
   logger.info(`TrainTime Server is listening on ${port}`);
+});*/
+
+const server = http.createServer(app);
+
+try {
+  const wss = new WebSocket.Server({ server });
+
+  const webSocket = new SocketHandler(busEmitter, wss);
+
+  twitter.updates(busEmitter);
+}
+catch (e) {
+  logger.error(e);
+}
+
+server.listen(port, (err) => {
+  if (err)
+    return logger.error('Server error:', err);
+
+  logger.info(`New Traintime server listening on ${ server.address().port}`);
 });

server/db/dbmanager.js

@@ -22,6 +22,34 @@ function prepareData(_obj) {
 
 exports.prepareData = prepareData;
 
+exports.vacuum = () => {
+  console.log('>> vacuum:');
+
+  return new Promise((resolve, reject) => {
+    db.run('VACUUM', function(err) {
+      if (err)
+        reject(err);
+
+      resolve({ 'msg':'Vacuum', 'changes': this.changes });
+    });
+  });
+};
+
+exports.deleteUpTo = (ms) => {
+  console.log('>> deleteUpTo:', ms);
+
+  const sql = 'delete from tweets where createdAt < ?';
+
+  return new Promise((resolve, reject) => {
+    db.run(sql, ms, function(err) {
+      if (err)
+        reject(err);
+
+      resolve({ 'msg':'Rows deleted', 'changes': this.changes });
+    });
+  });
+};
+
 exports.getAll = (list) => {
   console.log('>> getAll:', list);
   const outgoing = [];
@@ -43,6 +71,29 @@ exports.getAll = (list) => {
   });
 };
 
+exports.getLatestTweets = (ms) => {
+  console.log('>> getAll:', ms);
+  const outgoing = [];
+  // const sql = 'SELECT * FROM tweets order by _id desc limit 20';
+
+  const ts = (new Date().getTime() - ms) ;
+
+  const sql = 'SELECT * FROM tweets where ts>=? order by ts asc';
+
+  return new Promise((resolve, reject) => {
+    db.all(sql, [ts], (err, rows) => {
+      if (err)
+        reject(err);
+
+      rows.forEach((row) => {
+        outgoing.push(row);
+      });
+
+      resolve(outgoing) ;
+    });
+  });
+};
+
 exports.getTopOne = (hash) => {
   const sql = 'SELECT * FROM tweets order by _id desc limit 1';
 
@@ -56,7 +107,6 @@ exports.getTopOne = (hash) => {
   });
 };
 
-
 exports.getOne = (hash) => {
   const sql = 'SELECT * FROM menu WHERE hash  = ?';
 

server/lib/pubsub.js

@@ -0,0 +1,102 @@
+const logger = require('log4js').getLogger('pubsub');
+
+logger.level = 'debug';
+
+class PubSubManager {
+  constructor() {
+    this.channels = {
+      'twitter': {
+
+      },
+      'weather': {
+        'message': '',
+        'subscribers': []
+      },
+      'sports': {
+        'message': '',
+        'subscribers': []
+      }
+    };
+    this.brokerId = setInterval(() => {
+      this.broker(); 
+    }, 1000);
+    
+    this.updater = setInterval(() => {
+      this.doUpdater();
+    }, 60000);
+  }
+  subscribe(subscriber, channel) {
+    const channelSplit = channel.split('-');
+
+    if (channelSplit[0] === 't') {
+      if (!this.channels.twitter.hasOwnProperty(channel)) 
+        this.channels.twitter[channel] = {
+          'message': null,
+          'subscribers': []
+        };
+
+      logger.info(`subscribing to ${channel}`);
+
+      this.channels.twitter[channel].subscribers.push(subscriber);
+    }
+  }
+
+  unsubscribe(id) {
+    console.log('Forcing unsub for', id);
+    for (const channel in this.channels.twitter)
+      if (this.channels.twitter.hasOwnProperty(channel)) {
+        const channelObj = this.channels.twitter[channel];
+        channelObj.subscribers = channelObj.subscribers.filter(item => item.cuid !== id);
+      }
+  }
+
+  removeBroker() {
+    clearInterval(this.brokerId);
+  }
+
+  publish(publisher, channel, message) {
+    this.channels[channel].message = message;
+  }
+
+  publishTwitter( channel, message) {
+    // logger.debug('Publish:', channel, message);
+
+    if (!this.channels.twitter.hasOwnProperty(channel))
+      this.channels.twitter[channel] = {
+        'message': null,
+        'subscribers': []
+      };
+
+    this.channels.twitter[channel].message = message;
+  }
+
+  doUpdater() {
+    for (const channel in this.channels.twitter)
+      // logger.debug(channel);
+      if (this.channels.twitter.hasOwnProperty(channel)) {
+        const channelObj = this.channels.twitter[channel];
+        console.log(`Subscribers: ${channelObj.subscribers.length} in ${channel}`);
+        // console.log(channelObj.subscribers);
+      }
+  }
+  broker() {
+    // logger.debug('Broker...');
+    for (const channel in this.channels.twitter) 
+      // logger.debug(channel);
+      if (this.channels.twitter.hasOwnProperty(channel)) {
+        const channelObj = this.channels.twitter[channel];
+        if (channelObj.message) {
+          console.log(`found message: ${channelObj.message} in ${channel}`);
+
+          channelObj.subscribers.forEach(subscriber => {
+            subscriber.send(JSON.stringify({
+              'message': channelObj.message
+            }));
+          });
+
+          channelObj.message = '';
+        }
+      }
+  }
+}
+module.exports = PubSubManager;

server/lib/train.js

@@ -2,6 +2,7 @@
 const request = require('request');
 const http = require('http');
 const cheerio = require('cheerio');
+const crc = require('node-crc');
 // const STRING = require('string');
 const logger = require('log4js').getLogger('train');
 const trainCache = {
@@ -20,6 +21,15 @@ function getTrainTimes(req, res) {
     // logger.debug(`https://huxley.apphb.com${url}`);
 
     Query(function (a, b) {
+
+      if (a.hasOwnProperty('nrccMessages') && a.nrccMessages !== null) 
+        a.nrccMessages = a.nrccMessages.map((item) => {
+          const crcid = crc.crc64(Buffer.from(item.value, 'utf8')).toString('hex');
+          item.id = `nrcc-${crcid}`;
+
+          return item;
+        });
+      
       res.setHeader('Content-Type', 'application/json');
 
       res.end(JSON.stringify(a));

server/lib/twitter.js

@@ -5,9 +5,36 @@
  * Time: 16:19
 
  */
-
+const logger = require('log4js').getLogger('Twitter');
 const dbmanager = require('../db/dbmanager');
 
+let intervalID = 0;
+let eventEmitter;
+logger.level = 'debug';
+
+function doTweetUpdates() {
+  logger.info('doTweetUpdates');
+
+  dbmanager.getLatestTweets(10000)
+    .then((data) => {
+      // console.log(data);
+
+      data.forEach((item) => {
+        logger.debug(`t-${item.userIDStr}`);
+        eventEmitter.emit('sendTweet', `t-${item.userIDStr}`, item);
+      });
+    })
+    .catch((err) => {
+      console.error(err.message);
+    });
+}
+
+function startTweetUpdates() {
+  intervalID = setInterval(() => {
+    doTweetUpdates();
+  }, 10000);
+}
+
 exports.top = (req, res) => {
   dbmanager.getTopOne()
     .then((data) => {
@@ -32,3 +59,9 @@ exports.all = (req, res) => {
       });
     });
 };
+
+exports.updates = (newEmitter) => {
+  logger.info('Setting events');
+  eventEmitter = newEmitter;
+  startTweetUpdates();
+};

server/lib/wshandlerv2.js

@@ -0,0 +1,50 @@
+/**
+ *
+ * User: Martin Donnelly
+ * Date: 2016-09-07
+ * Time: 15:33
+ *
+ */
+const url = require('url');
+const logger = require('log4js').getLogger();
+
+module.exports = function(events, wsServer) {
+  'use strict';
+
+  logger.debug('>> new WS', wsServer);
+  wsServer.on('connection', function connection(ws) {
+    // console.log('>> WS:', ws);
+    // const location = url.parse(ws.upgradeReq.url, true);
+
+    logger.info('Creating event for this connection');
+
+    logger.info(`${new Date() } Connection accepted.`);
+
+    const sendSocketHandler = (obj) => {
+      logger.debug('sendSocketHandler', obj);
+      try {
+        ws.send(JSON.stringify(obj));
+      }
+      catch (err) {
+        logger.error(err);
+        logger.warn('Offending object: ', obj);
+      }
+    };
+
+    events.on('sendSocket', sendSocketHandler);
+
+    ws.on('message', function(message) {
+      console.log('received:', message);
+
+      if (message === 'update') 
+        events.emit('update');
+    });
+
+    ws.on('close', function(reasonCode, description) {
+      logger.info(`${new Date() } Peer ${ connection.remoteAddress } disconnected.`);
+      events.removeListener('sendSocket', sendSocketHandler);
+    });
+  });
+
+  return module;
+};

server/lib/wshandlerv3.js

@@ -0,0 +1,78 @@
+/**
+ *
+ * User: Martin Donnelly
+ * Date: 2016-09-07
+ * Time: 15:33
+ *
+ */
+
+// from
+// https://medium.com/unprogrammer/implementing-publisher-subscriber-pattern-using-javascript-nodejs-and-websockets-82036da7e174
+
+const url = require('url');
+const cuid = require('cuid');
+
+const PubSubManager = require('./pubsub');
+const logger = require('log4js').getLogger('wshandlerv3');
+logger.level = 'debug';
+
+const pubSubManager = new PubSubManager();
+
+function noop() {}
+
+function heartbeat() {
+  this.isAlive = true;
+}
+
+module.exports = function(events, wsServer) {
+  'use strict';
+
+  logger.debug('>> new WS', wsServer);
+  wsServer.on('connection', (ws, req) => {
+    ws.cuid = cuid();
+    console.log(`Connection request for ${ws.cuid}`);
+
+    ws.isAlive = true;
+    ws.on('pong', heartbeat);
+    
+    ws.on('message', (data) => {
+      console.log(`data: ${ data}`, this);
+      const json = JSON.parse(data);
+      const request = json.request;
+      const message = json.message;
+      const channel = json.channel;
+
+      switch (request) {
+
+        case 'PUBLISH':
+          pubSubManager.publish(ws, channel, message);
+          break;
+        case 'SUBSCRIBE':
+          pubSubManager.subscribe(ws, channel);
+          break;
+      
+      }
+    });
+    ws.on('close', () => {
+      console.log('Stopping client connection.', ws.cuid);
+      pubSubManager.unsubscribe(ws.cuid);
+    });
+  });
+
+  const interval = setInterval(function ping() {
+    wsServer.clients.forEach(function each(ws) {
+      if (ws.isAlive === false) return ws.terminate();
+
+      ws.isAlive = false;
+      ws.ping(noop);
+    });
+  }, 30000);
+  
+  const sendTweetHandler = (id, obj) => {
+    pubSubManager.publishTwitter(id, obj);
+  };
+
+  events.on('sendTweet', sendTweetHandler);
+
+  return module;
+};

twitter.js

@@ -40,6 +40,7 @@ const accounts = [
   {'name':'ScotRail', 'id':61569136},
   {'name':'Stansted_Exp', 'id':257511611},
   {'name':'TfL', 'id':47319664},
+  {'name':'WestMidRailway', 'id':915554470175657984},
 
   {'name':'NetworkRailBHM', 'id':583910976},
   {'name':'NetworkRailEDB', 'id':586614081},
@@ -49,11 +50,37 @@ const accounts = [
   {'name':'NetworkRailLST', 'id':581826097},
   {'name':'NetworkRailMAN', 'id':583895871},
   {'name':'NetworkRailVIC', 'id':587354752},
+
+  {'name':'BTP', 'id':266094415},
+  {'name':'BTPAvonSomerset', 'id':738664125132345344},
+  {'name':'BTPBhm', 'id':952003488},
+  {'name':'BTPBlackCountry', 'id':767698362866999297},
+  {'name':'BTPCambs', 'id':2574726074},
+  {'name':'BTPCardiff_NWP', 'id':951714852},
+  {'name':'BTPEAnglia', 'id':4479942923},
+  {'name':'BTPEssex', 'id':2949032015},
+  {'name':'BTPGtrMcr', 'id':1670204977},
+  {'name':'BTPLeics', 'id':761147194598711296},
+  {'name':'BTPLiverpoolSt', 'id':951912242},
+  {'name':'BTPLondon', 'id':957226980},
+  {'name':'BTPLondonBridge', 'id':3346645594},
+  {'name':'BTPMersey', 'id':951748434},
+  {'name':'BTPNorthScot', 'id':2238888007},
+  {'name':'BTPNorthWales', 'id':951487338},
+  {'name':'BTPOxon', 'id':741228701791178753},
+  {'name':'BTPPontypridd', 'id':1672678292},
   {'name':'BTPScotland', 'id':957256160},
+  {'name':'BTPSouthYorks', 'id':3384315676},
+  {'name':'BTPTeesValley', 'id':802182849872936962},
+  {'name':'BTPUnderground', 'id':986236195049897985},
+  {'name':'BTPWales', 'id':1430734374},
+  {'name':'BTPWaterloo', 'id':951997044},
+  {'name':'BTPWestScot', 'id':951757261}
+
 
 ];
 
-
+// {'name':'', 'id':},
 (async function(){
  /* const response = await user.getBearerToken();
 
@@ -61,6 +88,29 @@ const accounts = [
     bearer_token: response.access_token
   });*/
 
+  const cleanDB = async function(){
+
+    const aDay = 86400000;
+    const now = new Date().getTime() ;
+
+    const oldDate = now - (15 * aDay);
+
+    console.log('Clean up to:', oldDate );
+
+    await dbmanager.deleteUpTo(oldDate).then((v) => {
+      console.log(v);
+    }).catch((err) => {
+      console.error(err);
+    });
+
+    await dbmanager.vacuum().then((v) => {
+      console.log(v);
+    }).catch((err) => {
+      console.error(err);
+    });
+
+  };
+
   const validAccounts = accounts.map(item => {
     return item.id;
   })
@@ -72,6 +122,8 @@ const accounts = [
 
   console.log('Starting:', parameters);
 
+  await cleanDB();
+
   const stream = client.stream("statuses/filter", parameters)
     .on("start", response => console.log('Started!'))
     .on("data", tweet => {

version.txt

@@ -0,0 +1 @@
+2.1.0