From 204d995a4a7fa57e6e83c4bb4567a6d639fab2f4 Mon Sep 17 00:00:00 2001 From: Martin Donnelly Date: Thu, 20 Aug 2020 15:08:07 +0100 Subject: [PATCH] Fixes and tidying up the server side of the sockets --- ecosystem.config.js | 11 +- server.js | 2 +- server/db/dbmanager.js | 14 +- server/lib/pubsub.js | 14 +- server/lib/train.js | 6 +- server/lib/twitter.js | 23 +++- server/lib/twitterList.json | 262 ++++++++++++++++++++++++++++++++++++ server/lib/wshandlerv3.js | 8 +- 8 files changed, 308 insertions(+), 32 deletions(-) create mode 100644 server/lib/twitterList.json diff --git a/ecosystem.config.js b/ecosystem.config.js index dab9398..cc597bd 100644 --- a/ecosystem.config.js +++ b/ecosystem.config.js @@ -1,14 +1,17 @@ module.exports = { - 'apps' : [{ + 'apps': [{ 'name': 'Traintimes Server', 'script': './server.js', - 'watch': ['./server.js', 'live/build'] + 'watch': ['./server.js', 'live/build'], + 'env': { + 'LOGGER_LEVEL': 'warn' + } }, { 'name': 'Twitter Grabber', 'script': './twitter.js', 'watch': './twitter.js', 'watch_delay': 10000, - 'cron_restart' : '0 4 * * */3' + 'cron_restart': '0 4 * * */3' }] - + }; diff --git a/server.js b/server.js index 8e71218..97f8f62 100644 --- a/server.js +++ b/server.js @@ -15,7 +15,7 @@ const SocketHandler = require('./server/lib/wshandlerv3'); const Events = require('events'); const busEmitter = new Events.EventEmitter(); -logger.level = 'debug'; +logger.level = process.env.LOGGER_LEVEL || 'debug'; const app = express(); const port = process.env.PORT || 8100; diff --git a/server/db/dbmanager.js b/server/db/dbmanager.js index 6d0d13a..1acdf87 100644 --- a/server/db/dbmanager.js +++ b/server/db/dbmanager.js @@ -1,7 +1,9 @@ +const logger = require('log4js').getLogger('DBManager'); const db = require('./connect'); const fecha = require('fecha'); +logger.level = process.env.LOGGER_LEVEL || 'debug'; // exports.create = (req, res) => { function prepareData(_obj) { @@ -23,7 +25,7 @@ function prepareData(_obj) { exports.prepareData = prepareData; exports.vacuum = () => { - console.log('>> vacuum:'); + logger.info('>> vacuum:'); return new Promise((resolve, reject) => { db.run('VACUUM', function(err) { @@ -36,7 +38,7 @@ exports.vacuum = () => { }; exports.deleteUpTo = (ms) => { - console.log('>> deleteUpTo:', ms); + logger.info('>> deleteUpTo:', ms); const sql = 'delete from tweets where createdAt < ?'; @@ -51,7 +53,7 @@ exports.deleteUpTo = (ms) => { }; exports.getAll = (list) => { - console.log('>> getAll:', list); + logger.info('>> getAll:', list); const outgoing = []; // const sql = 'SELECT * FROM tweets order by _id desc limit 20'; @@ -72,7 +74,7 @@ exports.getAll = (list) => { }; exports.getLatestTweets = (ms) => { - console.log('>> getAll:', ms); + logger.info('>> getAll:', ms); const outgoing = []; // const sql = 'SELECT * FROM tweets order by _id desc limit 20'; @@ -204,7 +206,7 @@ exports.getRandom = (timestamp) => { exports.updateTimestamps = (newTimestamp, items) => { const sqlTimestamp = ~~(newTimestamp / 1000); - console.log('>> items', items); + logger.info('>> items', items); const sql = 'UPDATE menu SET lastused = $lastused WHERE _id = $in'; @@ -213,7 +215,7 @@ exports.updateTimestamps = (newTimestamp, items) => { const stmt = db.prepare(sql); items.forEach((item) => { - console.log(item); + logger.info(item); const newData = { '$lastused':sqlTimestamp, '$in':item }; stmt.run(newData); diff --git a/server/lib/pubsub.js b/server/lib/pubsub.js index 93b55b4..4e60b97 100644 --- a/server/lib/pubsub.js +++ b/server/lib/pubsub.js @@ -1,6 +1,6 @@ const logger = require('log4js').getLogger('pubsub'); -logger.level = 'debug'; +logger.level = process.env.LOGGER_LEVEL || 'debug'; class PubSubManager { constructor() { @@ -21,9 +21,9 @@ class PubSubManager { this.broker(); }, 1000); - this.updater = setInterval(() => { + /* this.updater = setInterval(() => { this.doUpdater(); - }, 60000); + }, 60000);*/ } subscribe(subscriber, channel) { const channelSplit = channel.split('-'); @@ -42,7 +42,7 @@ class PubSubManager { } unsubscribe(id) { - console.log('Forcing unsub for', id); + logger.info('Forcing unsub for', id); for (const channel in this.channels.twitter) if (this.channels.twitter.hasOwnProperty(channel)) { const channelObj = this.channels.twitter[channel]; @@ -75,8 +75,8 @@ class PubSubManager { // logger.debug(channel); if (this.channels.twitter.hasOwnProperty(channel)) { const channelObj = this.channels.twitter[channel]; - console.log(`Subscribers: ${channelObj.subscribers.length} in ${channel}`); - // console.log(channelObj.subscribers); + logger.info(`Subscribers: ${channelObj.subscribers.length} in ${channel}`); + // logger.info(channelObj.subscribers); } } broker() { @@ -86,7 +86,7 @@ class PubSubManager { if (this.channels.twitter.hasOwnProperty(channel)) { const channelObj = this.channels.twitter[channel]; if (channelObj.message) { - console.log(`found message: ${channelObj.message} in ${channel}`); + logger.info(`found message: ${channelObj.message} in ${channel}`); channelObj.subscribers.forEach(subscriber => { subscriber.send(JSON.stringify({ diff --git a/server/lib/train.js b/server/lib/train.js index d4b6a7d..6c94de5 100644 --- a/server/lib/train.js +++ b/server/lib/train.js @@ -10,7 +10,7 @@ const trainCache = { 'data': {} }; -logger.level = 'info'; +logger.level = process.env.LOGGER_LEVEL || 'debug'; function getTrainTimes(req, res) { // console.log(req); @@ -207,12 +207,12 @@ function Query(callback, r, host, path) { callback(JSON.parse(data), r); }); response.on('error', function (e) { - console.error(e); + logger.error(e); }); }).end(); } catch (e) { - console.log(e); + logger.debug(e); } } diff --git a/server/lib/twitter.js b/server/lib/twitter.js index 8a00bd4..c8f3e53 100644 --- a/server/lib/twitter.js +++ b/server/lib/twitter.js @@ -7,28 +7,36 @@ */ const logger = require('log4js').getLogger('Twitter'); const dbmanager = require('../db/dbmanager'); +const rawTwitterList = require('./twitterList.json'); +let twitterList = []; let intervalID = 0; let eventEmitter; -logger.level = 'debug'; +logger.level = process.env.LOGGER_LEVEL || 'debug'; function doTweetUpdates() { logger.info('doTweetUpdates'); dbmanager.getLatestTweets(10000) .then((data) => { - // console.log(data); - data.forEach((item) => { - logger.debug(`t-${item.userIDStr}`); - eventEmitter.emit('sendTweet', `t-${item.userIDStr}`, item); + if(twitterList.indexOf(item.userID) !== -1) + eventEmitter.emit('sendTweet', `t-${item.userIDStr}`, item); }); }) .catch((err) => { - console.error(err.message); + logger.error(err.message); }); } +function buildList() { + twitterList = rawTwitterList.map((item) => { + return item.id; + }); + + logger.debug(twitterList); +} + function startTweetUpdates() { intervalID = setInterval(() => { doTweetUpdates(); @@ -61,7 +69,8 @@ exports.all = (req, res) => { }; exports.updates = (newEmitter) => { - logger.info('Setting events'); + logger.warn('Setting events'); eventEmitter = newEmitter; + buildList(); startTweetUpdates(); }; diff --git a/server/lib/twitterList.json b/server/lib/twitterList.json new file mode 100644 index 0000000..8d7d92f --- /dev/null +++ b/server/lib/twitterList.json @@ -0,0 +1,262 @@ +[ + { + "follow": false, + "name": "nationalrailenq", + "id": 33546465 + }, + { + "follow": false, + "name": "networkrail", + "id": 365344176 + }, + { + "follow": false, + "name": "NetworkRailSCOT", + "id": 402687948 + }, + { + "follow": false, + "name": "AvantiWestCoast", + "id": 1143560758476906497 + }, + { + "follow": false, + "name": "CalSleeper", + "id": 2870293725 + }, + { + "follow": false, + "name": "CrossCountryUK", + "id": 153368708 + }, + { + "follow": false, + "name": "Eurostar", + "id": 98412169 + }, + { + "follow": false, + "name": "EurostarUK", + "id": 59742254 + }, + { + "follow": false, + "name": "GatwickExpress", + "id": 163816182 + }, + { + "follow": false, + "name": "GlasgowSubway", + "id": 224607925 + }, + { + "follow": false, + "name": "GWRHelp", + "id": 15589815 + }, + { + "follow": false, + "name": "HeathrowExpress", + "id": 20240678 + }, + { + "follow": false, + "name": "LNER", + "id": 313306238 + }, + { + "follow": false, + "name": "LNRailway", + "id": 910487328627535872 + }, + { + "follow": false, + "name": "northernassist", + "id": 194512268 + }, + { + "follow": false, + "name": "ScotRail", + "id": 61569136 + }, + { + "follow": false, + "name": "Stansted_Exp", + "id": 257511611 + }, + { + "follow": false, + "name": "TfL", + "id": 47319664 + }, + { + "follow": false, + "name": "WestMidRailway", + "id": 915554470175657984 + }, + { + "follow": false, + "name": "NetworkRailBHM", + "id": 583910976 + }, + { + "follow": false, + "name": "NetworkRailEDB", + "id": 586614081 + }, + { + "follow": false, + "name": "NetworkRailEUS", + "id": 581807264 + }, + { + "follow": false, + "name": "NetworkRailGLC", + "id": 421061171 + }, + { + "follow": false, + "name": "NetworkRailKGX", + "id": 459192871 + }, + { + "follow": false, + "name": "NetworkRailLST", + "id": 581826097 + }, + { + "follow": false, + "name": "NetworkRailMAN", + "id": 583895871 + }, + { + "follow": false, + "name": "NetworkRailVIC", + "id": 587354752 + }, + { + "follow": false, + "name": "BTP", + "id": 266094415 + }, + { + "follow": false, + "name": "BTPAvonSomerset", + "id": 738664125132345344 + }, + { + "follow": false, + "name": "BTPBhm", + "id": 952003488 + }, + { + "follow": false, + "name": "BTPBlackCountry", + "id": 767698362866999297 + }, + { + "follow": false, + "name": "BTPCambs", + "id": 2574726074 + }, + { + "follow": false, + "name": "BTPCardiff_NWP", + "id": 951714852 + }, + { + "follow": false, + "name": "BTPEAnglia", + "id": 4479942923 + }, + { + "follow": false, + "name": "BTPEssex", + "id": 2949032015 + }, + { + "follow": false, + "name": "BTPGtrMcr", + "id": 1670204977 + }, + { + "follow": false, + "name": "BTPLeics", + "id": 761147194598711296 + }, + { + "follow": false, + "name": "BTPLiverpoolSt", + "id": 951912242 + }, + { + "follow": false, + "name": "BTPLondon", + "id": 957226980 + }, + { + "follow": false, + "name": "BTPLondonBridge", + "id": 3346645594 + }, + { + "follow": false, + "name": "BTPMersey", + "id": 951748434 + }, + { + "follow": false, + "name": "BTPNorthScot", + "id": 2238888007 + }, + { + "follow": false, + "name": "BTPNorthWales", + "id": 951487338 + }, + { + "follow": false, + "name": "BTPOxon", + "id": 741228701791178753 + }, + { + "follow": false, + "name": "BTPPontypridd", + "id": 1672678292 + }, + { + "follow": false, + "name": "BTPScotland", + "id": 957256160 + }, + { + "follow": false, + "name": "BTPSouthYorks", + "id": 3384315676 + }, + { + "follow": false, + "name": "BTPTeesValley", + "id": 802182849872936962 + }, + { + "follow": false, + "name": "BTPUnderground", + "id": 986236195049897985 + }, + { + "follow": false, + "name": "BTPWales", + "id": 1430734374 + }, + { + "follow": false, + "name": "BTPWaterloo", + "id": 951997044 + }, + { + "follow": false, + "name": "BTPWestScot", + "id": 951757261 + } +] diff --git a/server/lib/wshandlerv3.js b/server/lib/wshandlerv3.js index 7add8ab..43f9bcc 100644 --- a/server/lib/wshandlerv3.js +++ b/server/lib/wshandlerv3.js @@ -14,7 +14,7 @@ const cuid = require('cuid'); const PubSubManager = require('./pubsub'); const logger = require('log4js').getLogger('wshandlerv3'); -logger.level = 'debug'; +logger.level = process.env.LOGGER_LEVEL || 'debug'; const pubSubManager = new PubSubManager(); @@ -30,13 +30,13 @@ module.exports = function(events, wsServer) { logger.debug('>> new WS', wsServer); wsServer.on('connection', (ws, req) => { ws.cuid = cuid(); - console.log(`Connection request for ${ws.cuid}`); + logger.debug(`Connection request for ${ws.cuid}`); ws.isAlive = true; ws.on('pong', heartbeat); ws.on('message', (data) => { - console.log(`data: ${ data}`, this); + logger.debug(`data: ${ data}`, this); const json = JSON.parse(data); const request = json.request; const message = json.message; @@ -54,7 +54,7 @@ module.exports = function(events, wsServer) { } }); ws.on('close', () => { - console.log('Stopping client connection.', ws.cuid); + logger.debug('Stopping client connection.', ws.cuid); pubSubManager.unsubscribe(ws.cuid); }); });