Merge branch 'ttpwa-004' into 'development'

Fixes and tidying up the server side of the sockets

See merge request martind2000/traintimesPWA!10

ecosystem.config.js

@@ -1,14 +1,17 @@
 module.exports = {
-  'apps' : [{
+  'apps': [{
     'name': 'Traintimes Server',
     'script': './server.js',
-    'watch': ['./server.js', 'live/build']
+    'watch': ['./server.js', 'live/build'],
+    'env': {
+      'LOGGER_LEVEL': 'warn'
+    }
   }, {
     'name': 'Twitter Grabber',
     'script': './twitter.js',
     'watch': './twitter.js',
     'watch_delay': 10000,
-    'cron_restart' : '0 4 * * */3'
+    'cron_restart': '0 4 * * */3'
   }]
-  
+
 };

server.js

@@ -15,7 +15,7 @@ const SocketHandler = require('./server/lib/wshandlerv3');
 const Events = require('events');
 const busEmitter = new Events.EventEmitter();
 
-logger.level = 'debug';
+logger.level = process.env.LOGGER_LEVEL || 'debug';
 
 const app = express();
 const port = process.env.PORT || 8100;

server/db/dbmanager.js

@@ -1,7 +1,9 @@
+const logger = require('log4js').getLogger('DBManager');
 const db = require('./connect');
 
 const fecha = require('fecha');
 
+logger.level = process.env.LOGGER_LEVEL || 'debug';
 // exports.create = (req, res) => {
 
 function prepareData(_obj) {
@@ -23,7 +25,7 @@ function prepareData(_obj) {
 exports.prepareData = prepareData;
 
 exports.vacuum = () => {
-  console.log('>> vacuum:');
+  logger.info('>> vacuum:');
 
   return new Promise((resolve, reject) => {
     db.run('VACUUM', function(err) {
@@ -36,7 +38,7 @@ exports.vacuum = () => {
 };
 
 exports.deleteUpTo = (ms) => {
-  console.log('>> deleteUpTo:', ms);
+  logger.info('>> deleteUpTo:', ms);
 
   const sql = 'delete from tweets where createdAt < ?';
 
@@ -51,7 +53,7 @@ exports.deleteUpTo = (ms) => {
 };
 
 exports.getAll = (list) => {
-  console.log('>> getAll:', list);
+  logger.info('>> getAll:', list);
   const outgoing = [];
   // const sql = 'SELECT * FROM tweets order by _id desc limit 20';
 
@@ -72,7 +74,7 @@ exports.getAll = (list) => {
 };
 
 exports.getLatestTweets = (ms) => {
-  console.log('>> getAll:', ms);
+  logger.info('>> getAll:', ms);
   const outgoing = [];
   // const sql = 'SELECT * FROM tweets order by _id desc limit 20';
 
@@ -204,7 +206,7 @@ exports.getRandom = (timestamp) => {
 exports.updateTimestamps = (newTimestamp, items) => {
   const sqlTimestamp = ~~(newTimestamp / 1000);
 
-  console.log('>> items', items);
+  logger.info('>> items', items);
   
   const sql = 'UPDATE menu SET lastused = $lastused WHERE _id = $in';
 
@@ -213,7 +215,7 @@ exports.updateTimestamps = (newTimestamp, items) => {
       const stmt = db.prepare(sql);
 
       items.forEach((item) => {
-        console.log(item);
+        logger.info(item);
         const newData = { '$lastused':sqlTimestamp, '$in':item };
 
         stmt.run(newData);

server/lib/pubsub.js

@@ -1,6 +1,6 @@
 const logger = require('log4js').getLogger('pubsub');
 
-logger.level = 'debug';
+logger.level = process.env.LOGGER_LEVEL || 'debug';
 
 class PubSubManager {
   constructor() {
@@ -21,9 +21,9 @@ class PubSubManager {
       this.broker(); 
     }, 1000);
     
-    this.updater = setInterval(() => {
+    /* this.updater = setInterval(() => {
       this.doUpdater();
-    }, 60000);
+    }, 60000);*/
   }
   subscribe(subscriber, channel) {
     const channelSplit = channel.split('-');
@@ -42,7 +42,7 @@ class PubSubManager {
   }
 
   unsubscribe(id) {
-    console.log('Forcing unsub for', id);
+    logger.info('Forcing unsub for', id);
     for (const channel in this.channels.twitter)
       if (this.channels.twitter.hasOwnProperty(channel)) {
         const channelObj = this.channels.twitter[channel];
@@ -75,8 +75,8 @@ class PubSubManager {
       // logger.debug(channel);
       if (this.channels.twitter.hasOwnProperty(channel)) {
         const channelObj = this.channels.twitter[channel];
-        console.log(`Subscribers: ${channelObj.subscribers.length} in ${channel}`);
+        logger.info(`Subscribers: ${channelObj.subscribers.length} in ${channel}`);
-        // console.log(channelObj.subscribers);
+        // logger.info(channelObj.subscribers);
       }
   }
   broker() {
@@ -86,7 +86,7 @@ class PubSubManager {
       if (this.channels.twitter.hasOwnProperty(channel)) {
         const channelObj = this.channels.twitter[channel];
         if (channelObj.message) {
-          console.log(`found message: ${channelObj.message} in ${channel}`);
+          logger.info(`found message: ${channelObj.message} in ${channel}`);
 
           channelObj.subscribers.forEach(subscriber => {
             subscriber.send(JSON.stringify({

server/lib/train.js

@@ -10,7 +10,7 @@ const trainCache = {
   'data': {}
 };
 
-logger.level = 'info';
+logger.level = process.env.LOGGER_LEVEL || 'debug';
 
 function getTrainTimes(req, res) {
   // console.log(req);
@@ -207,12 +207,12 @@ function Query(callback, r, host, path) {
         callback(JSON.parse(data), r);
       });
       response.on('error', function (e) {
-        console.error(e);
+        logger.error(e);
       });
     }).end();
   }
   catch (e) {
-    console.log(e);
+    logger.debug(e);
   }
 }
 

server/lib/twitter.js

@@ -7,28 +7,36 @@
  */
 const logger = require('log4js').getLogger('Twitter');
 const dbmanager = require('../db/dbmanager');
+const rawTwitterList = require('./twitterList.json');
 
+let twitterList = [];
 let intervalID = 0;
 let eventEmitter;
-logger.level = 'debug';
+logger.level = process.env.LOGGER_LEVEL || 'debug';
 
 function doTweetUpdates() {
   logger.info('doTweetUpdates');
 
   dbmanager.getLatestTweets(10000)
     .then((data) => {
-      // console.log(data);
-
       data.forEach((item) => {
-        logger.debug(`t-${item.userIDStr}`);
+        if(twitterList.indexOf(item.userID) !== -1)
-        eventEmitter.emit('sendTweet', `t-${item.userIDStr}`, item);
+          eventEmitter.emit('sendTweet', `t-${item.userIDStr}`, item);
       });
     })
     .catch((err) => {
-      console.error(err.message);
+      logger.error(err.message);
     });
 }
 
+function buildList() {
+  twitterList = rawTwitterList.map((item) => {
+    return item.id;
+  });
+
+  logger.debug(twitterList);
+}
+
 function startTweetUpdates() {
   intervalID = setInterval(() => {
     doTweetUpdates();
@@ -61,7 +69,8 @@ exports.all = (req, res) => {
 };
 
 exports.updates = (newEmitter) => {
-  logger.info('Setting events');
+  logger.warn('Setting events');
   eventEmitter = newEmitter;
+  buildList();
   startTweetUpdates();
 };

server/lib/twitterList.json

@@ -0,0 +1,262 @@
+[
+  {
+    "follow": false,
+    "name": "nationalrailenq",
+    "id": 33546465
+  },
+  {
+    "follow": false,
+    "name": "networkrail",
+    "id": 365344176
+  },
+  {
+    "follow": false,
+    "name": "NetworkRailSCOT",
+    "id": 402687948
+  },
+  {
+    "follow": false,
+    "name": "AvantiWestCoast",
+    "id": 1143560758476906497
+  },
+  {
+    "follow": false,
+    "name": "CalSleeper",
+    "id": 2870293725
+  },
+  {
+    "follow": false,
+    "name": "CrossCountryUK",
+    "id": 153368708
+  },
+  {
+    "follow": false,
+    "name": "Eurostar",
+    "id": 98412169
+  },
+  {
+    "follow": false,
+    "name": "EurostarUK",
+    "id": 59742254
+  },
+  {
+    "follow": false,
+    "name": "GatwickExpress",
+    "id": 163816182
+  },
+  {
+    "follow": false,
+    "name": "GlasgowSubway",
+    "id": 224607925
+  },
+  {
+    "follow": false,
+    "name": "GWRHelp",
+    "id": 15589815
+  },
+  {
+    "follow": false,
+    "name": "HeathrowExpress",
+    "id": 20240678
+  },
+  {
+    "follow": false,
+    "name": "LNER",
+    "id": 313306238
+  },
+  {
+    "follow": false,
+    "name": "LNRailway",
+    "id": 910487328627535872
+  },
+  {
+    "follow": false,
+    "name": "northernassist",
+    "id": 194512268
+  },
+  {
+    "follow": false,
+    "name": "ScotRail",
+    "id": 61569136
+  },
+  {
+    "follow": false,
+    "name": "Stansted_Exp",
+    "id": 257511611
+  },
+  {
+    "follow": false,
+    "name": "TfL",
+    "id": 47319664
+  },
+  {
+    "follow": false,
+    "name": "WestMidRailway",
+    "id": 915554470175657984
+  },
+  {
+    "follow": false,
+    "name": "NetworkRailBHM",
+    "id": 583910976
+  },
+  {
+    "follow": false,
+    "name": "NetworkRailEDB",
+    "id": 586614081
+  },
+  {
+    "follow": false,
+    "name": "NetworkRailEUS",
+    "id": 581807264
+  },
+  {
+    "follow": false,
+    "name": "NetworkRailGLC",
+    "id": 421061171
+  },
+  {
+    "follow": false,
+    "name": "NetworkRailKGX",
+    "id": 459192871
+  },
+  {
+    "follow": false,
+    "name": "NetworkRailLST",
+    "id": 581826097
+  },
+  {
+    "follow": false,
+    "name": "NetworkRailMAN",
+    "id": 583895871
+  },
+  {
+    "follow": false,
+    "name": "NetworkRailVIC",
+    "id": 587354752
+  },
+  {
+    "follow": false,
+    "name": "BTP",
+    "id": 266094415
+  },
+  {
+    "follow": false,
+    "name": "BTPAvonSomerset",
+    "id": 738664125132345344
+  },
+  {
+    "follow": false,
+    "name": "BTPBhm",
+    "id": 952003488
+  },
+  {
+    "follow": false,
+    "name": "BTPBlackCountry",
+    "id": 767698362866999297
+  },
+  {
+    "follow": false,
+    "name": "BTPCambs",
+    "id": 2574726074
+  },
+  {
+    "follow": false,
+    "name": "BTPCardiff_NWP",
+    "id": 951714852
+  },
+  {
+    "follow": false,
+    "name": "BTPEAnglia",
+    "id": 4479942923
+  },
+  {
+    "follow": false,
+    "name": "BTPEssex",
+    "id": 2949032015
+  },
+  {
+    "follow": false,
+    "name": "BTPGtrMcr",
+    "id": 1670204977
+  },
+  {
+    "follow": false,
+    "name": "BTPLeics",
+    "id": 761147194598711296
+  },
+  {
+    "follow": false,
+    "name": "BTPLiverpoolSt",
+    "id": 951912242
+  },
+  {
+    "follow": false,
+    "name": "BTPLondon",
+    "id": 957226980
+  },
+  {
+    "follow": false,
+    "name": "BTPLondonBridge",
+    "id": 3346645594
+  },
+  {
+    "follow": false,
+    "name": "BTPMersey",
+    "id": 951748434
+  },
+  {
+    "follow": false,
+    "name": "BTPNorthScot",
+    "id": 2238888007
+  },
+  {
+    "follow": false,
+    "name": "BTPNorthWales",
+    "id": 951487338
+  },
+  {
+    "follow": false,
+    "name": "BTPOxon",
+    "id": 741228701791178753
+  },
+  {
+    "follow": false,
+    "name": "BTPPontypridd",
+    "id": 1672678292
+  },
+  {
+    "follow": false,
+    "name": "BTPScotland",
+    "id": 957256160
+  },
+  {
+    "follow": false,
+    "name": "BTPSouthYorks",
+    "id": 3384315676
+  },
+  {
+    "follow": false,
+    "name": "BTPTeesValley",
+    "id": 802182849872936962
+  },
+  {
+    "follow": false,
+    "name": "BTPUnderground",
+    "id": 986236195049897985
+  },
+  {
+    "follow": false,
+    "name": "BTPWales",
+    "id": 1430734374
+  },
+  {
+    "follow": false,
+    "name": "BTPWaterloo",
+    "id": 951997044
+  },
+  {
+    "follow": false,
+    "name": "BTPWestScot",
+    "id": 951757261
+  }
+]

server/lib/wshandlerv3.js

@@ -14,7 +14,7 @@ const cuid = require('cuid');
 
 const PubSubManager = require('./pubsub');
 const logger = require('log4js').getLogger('wshandlerv3');
-logger.level = 'debug';
+logger.level = process.env.LOGGER_LEVEL || 'debug';
 
 const pubSubManager = new PubSubManager();
 
@@ -30,13 +30,13 @@ module.exports = function(events, wsServer) {
   logger.debug('>> new WS', wsServer);
   wsServer.on('connection', (ws, req) => {
     ws.cuid = cuid();
-    console.log(`Connection request for ${ws.cuid}`);
+    logger.debug(`Connection request for ${ws.cuid}`);
 
     ws.isAlive = true;
     ws.on('pong', heartbeat);
     
     ws.on('message', (data) => {
-      console.log(`data: ${ data}`, this);
+      logger.debug(`data: ${ data}`, this);
       const json = JSON.parse(data);
       const request = json.request;
       const message = json.message;
@@ -54,7 +54,7 @@ module.exports = function(events, wsServer) {
       }
     });
     ws.on('close', () => {
-      console.log('Stopping client connection.', ws.cuid);
+      logger.debug('Stopping client connection.', ws.cuid);
       pubSubManager.unsubscribe(ws.cuid);
     });
   });