Twitter grabber

Tweets added to server
New svelte front end

db/init_db.js

@@ -0,0 +1,107 @@
+print('------------------- Initialising Database Collections and Indexes --------------------------------');
+
+/**
+ * Mongo shell defaults to the 'test' database unless told otherwise.  So explictly choose the MDB database.
+ */
+db = db.getSiblingDB('TWITTER');
+print('Creating collections in:');
+printjson(db);
+print('Existing collections:');
+printjson(db.getCollectionNames());
+
+print('Adding collections.  This may take some time...');
+
+db.createCollection('Tweets');
+db.Account.dropIndexes();
+db.Account.createIndex({ 'TweetID':1 }, { 'unique':true, 'sparse':true, 'name':'getTweetByID' });
+
+/*
+
+db.createCollection('AccountArchive');
+db.AccountArchive.dropIndexes();
+db.AccountArchive.createIndex({'ClientID':1},{'unique':false,'sparse':true,'name':'getAccountByClientID'});
+
+db.createCollection('Address');
+db.Address.dropIndexes();
+db.Address.createIndex({'ClientID':1},{'unique':false,'sparse':true,'name':'getAddressesForClientID'});
+db.Address.createIndex({'_id':1,'ClientID':1},{'unique':true,'sparse':true,'name':'getAddressForClientID'});
+
+db.createCollection('AddressArchive');
+db.AddressArchive.dropIndexes();
+db.AddressArchive.createIndex({'ClientID':1},{'unique':false,'sparse':true,'name':'getAddressArchiveForClientID'});
+
+db.createCollection('BridgeLogin');
+db.BridgeLogin.dropIndexes();
+
+db.createCollection('Client');
+db.Client.dropIndexes();
+db.Client.createIndex({'ClientID':1},{'unique':true,'sparse':true,'name':'getClientByClientID'});
+db.Client.createIndex({'ClientName':1},{'unique':true,'sparse':true,'name':'getClientByClientName'});
+
+db.createCollection('ClientArchive');
+db.ClientArchive.dropIndexes();
+db.ClientArchive.createIndex({'ClientID':1},{'unique':false,'sparse':true,'name':'getClientByClientID'});
+
+db.createCollection('Device');
+db.Device.dropIndexes();
+db.Device.createIndex({'DeviceToken':1},{'unique':true,'name':'getDevicesByToken'});
+db.Device.createIndex({'DeviceNumber':1},{'unique':true,'sparse':true,'name':'getDevicesByNumber'});
+db.Device.createIndex({'ClientID':1},{'unique':false,'name':'getDevicesByClientID'});
+
+db.createCollection('DeviceArchive');
+db.DeviceArchive.dropIndexes();
+db.DeviceArchive.createIndex({'ClientID':1},{'unique':false,'name':'getDevicesByClientID'});
+
+db.createCollection('Images');
+db.Images.dropIndexes();
+db.Images.createIndex({'ClientID':1},{'unique':false,'sparse':true,'name':'getImagesByClientID'});
+
+db.createCollection('Items');
+db.Items.dropIndexes();
+db.Items.createIndex({'ClientID':1},{'unique':false,'sparse':true,'name':'getItemsByClientID'});
+
+db.createCollection('Messages');
+db.Messages.dropIndexes();
+db.Messages.createIndex({'ClientID':1,'TimeFilter':1},{'unique':false,'sparse':true,'name':'getMessagesByClientIDandTime'});
+
+db.createCollection('MessagesArchive');
+db.MessagesArchive.dropIndexes();
+db.MessagesArchive.createIndex({'ClientID':1},{'unique':false,'sparse':true,'name':'getMessagesByClientID'});
+
+db.createCollection('PayCode');
+db.PayCode.dropIndexes();
+db.PayCode.createIndex({'PayCode':1},{'unique':true,'sparse':true,'name':'getCurrentPayCodes'});
+db.PayCode.createIndex({'Expiry':1},{expireAfterSeconds:0,'name':'payCodeTTL'});
+
+db.createCollection('SystemLog');
+db.SystemLog.dropIndexes();
+db.SystemLog.createIndex({'DateTime':-1},{'unique':false,'sparse':true,'name':'getEntryByTime'});
+
+db.createCollection('Transaction');
+db.Transaction.dropIndexes();
+db.Transaction.createIndex({'MerchantInvoiceNumber.InvoiceNumber':-1,'MerchantInvoiceNumber.MerchantID':-1,'MerchantInvoiceNumber.MerchantIndex':1},{'unique':true,partialFilterExpression:{MerchantInvoiceNumber:{$exists:true}},name:'MerchantInvoiceNumberIndex'});
+
+db.createCollection('TransactionArchive');
+db.TransactionArchive.dropIndexes();
+db.TransactionArchive.createIndex({'CustomerClientID':1},{'unique':false,'sparse':true,'name':'getTransactionsByClientID'});
+
+db.createCollection('TransactionHistory');
+db.TransactionHistory.dropIndexes();
+db.TransactionHistory.createIndex({'ClientID':1,'AccountID':1,'SaleTime':1},{'unique':false,'sparse':true,'name':'getHistoryByClient'});
+db.TransactionHistory.createIndex({'AccountID':1},{'unique':false,'sparse':true,'name':'getHistoryByAccount'});
+db.TransactionHistory.createIndex({'OtherImage':1},{'unique':false,'sparse':true,'name':'findImage'});
+
+db.createCollection('TwoFARequests');
+db.TwoFARequests.dropIndexes();
+db.TwoFARequests.createIndex({'RequestExpiry':1},{'unique':false,expireAfterSeconds:0,'name':'requestTTL'});
+
+db.createCollection('WebConsoleSessions');
+db.WebConsoleSessions.dropIndexes();
+db.WebConsoleSessions.createIndex({'expires': 1}, {expireAfterSeconds: 60, 'name': 'webConsoleSessionsTTL'});
+*/
+
+print('Collections added.');
+print('Updated Collections:');
+printjson(db.getCollectionNames());
+
+print('------------------- DONE Initialising Database --------------------------------');

package.json

@@ -16,14 +16,18 @@
     "browserify": "^14.5.0",
     "compressjs": "^1.0.3",
     "cors": "^2.8.4",
+    "dotenv": "^8.2.0",
     "es6-promise": "^4.1.1",
     "express": "^4.16.2",
+    "fecha": "^4.2.0",
     "jquery": "^3.2.1",
     "json-lzw": "^0.3.2",
     "log4js": "^2.4.1",
     "lz77": "^1.1.0",
     "minibus": "^3.1.0",
     "muicss": "^0.9.33",
+    "sqlite3": "^4.2.0",
+    "twitter-lite": "^0.11.0",
     "underscore": "^1.8.3"
   },
   "devDependencies": {
@@ -66,6 +70,8 @@
     "mocha": "^3.5.3",
     "require-dir": "^0.3.2",
     "sinon": "^4.1.1",
+    "tape": "^5.0.0",
+    "tape-promise": "^4.0.0",
     "vinyl-buffer": "^1.0.0",
     "vinyl-source-stream": "^1.1.0",
     "watchify": "^3.9.0"

server.js

@@ -6,6 +6,7 @@ const apicache = require('apicache');
 const logger = require('log4js').getLogger('Server');
 
 const train = require('./server/lib/train');
+const twitter = require('./server/lib/twitter');
 logger.level = 'debug';
 
 const app = express();
@@ -15,6 +16,7 @@ const sitePath = 'live';
 
 apicache.options({ 'debug': true });
 const cache = apicache.middleware;
+app.use(express.json());
 
 app.use(express.static(path.join(__dirname, sitePath)));
 
@@ -24,6 +26,9 @@ app.use('/gettrains', cors(), cache('60 seconds'), train.getTrainTimes);
 app.use('/getnexttraintimes', cors(), cache('60 seconds'), train.getNextTrainTimes);
 app.use('/getroute', cors(), cache('60 seconds'), train.getRoute);
 app.use('/getnews', cors(), cache('30 minutes'), train.getNews);
+app.use('/getservice', cors(), cache('60 seconds'), train.getService);
+
+app.use('/twitter', cors(), twitter.all);
 
 process.on('uncaughtException', err => {
   logger.error(err);

server/db/connect.js

@@ -0,0 +1,5 @@
+const sqlite3 = require('sqlite3').verbose();
+console.log(`${__dirname}/../../db/twitter.db`);
+const db = new sqlite3.Database(`${__dirname}/../../db/twitter.db`);
+
+module.exports = db;

server/db/dbmanager.js

@@ -0,0 +1,180 @@
+const db = require('./connect');
+
+const fecha = require('fecha');
+
+// exports.create = (req, res) => {
+
+function prepareData(_obj) {
+  const newObj = Object.assign({}, { 'tweet':JSON.stringify(_obj) });
+
+  newObj.ts = parseInt(_obj.timestamp_ms, 10);
+
+  newObj.createdAt = fecha.parse(_obj.created_at, 'ddd MMM DD HH:mm:ss ZZ YYYY').getTime(); // Thu Apr 30 11:59:47 +0000 2020
+  newObj.id = _obj.id;
+  newObj.idStr = _obj.id_str;
+  newObj.userID = _obj.user.id;
+  newObj.userIDStr = _obj.user.id_str;
+  newObj.name = _obj.user.name;
+  newObj.screenName = _obj.user.screen_name;
+
+  return newObj;
+}
+
+exports.prepareData = prepareData;
+
+exports.getAll = (list) => {
+  console.log('>> getAll:', list);
+  const outgoing = [];
+  // const sql = 'SELECT * FROM tweets order by _id desc limit 20';
+
+  const sql = `SELECT * FROM tweets where userID In ( ${list.join(',')} ) order by _id desc limit 20`;
+
+  return new Promise((resolve, reject) => {
+    db.all(sql, [], (err, rows) => {
+      if (err)
+        reject(err);
+
+      rows.forEach((row) => {
+        outgoing.push(row);
+      });
+
+      resolve(outgoing) ;
+    });
+  });
+};
+
+exports.getTopOne = (hash) => {
+  const sql = 'SELECT * FROM tweets order by _id desc limit 1';
+
+  return new Promise((resolve, reject) => {
+    db.get(sql, [hash], (err, row) => {
+      if (err)
+        reject(err);
+
+      if (!err) resolve([row]);
+    });
+  });
+};
+
+
+exports.getOne = (hash) => {
+  const sql = 'SELECT * FROM menu WHERE hash  = ?';
+
+  return new Promise((resolve, reject) => {
+    db.get(sql, [hash], (err, row) => {
+      if (err)
+        reject(err);
+
+      if (!err) resolve(row);
+    });
+  });
+};
+
+exports.getOneShort = (short) => {
+  const sql = 'SELECT * FROM menu WHERE short  = ?';
+
+  return new Promise((resolve, reject) => {
+    db.get(sql, [short], (err, row) => {
+      if (err)
+        reject(err);
+
+      if (!err) resolve(row);
+    });
+  });
+};
+
+exports.insertOne = (data) => {
+  const sql = 'INSERT INTO tweets VALUES (?,?,?,?,?,?,?,?,?,?)';
+
+  const tweetObj = prepareData(data);
+
+  return new Promise((resolve, reject) => {
+    db.run(sql, [null, tweetObj.ts, tweetObj.createdAt, tweetObj.id, tweetObj.idStr, tweetObj.userID, tweetObj.userIDStr, tweetObj.name, tweetObj.screenName, tweetObj.tweet], function(err) {
+      if (err)
+        reject(err);
+
+      resolve({ 'msg':'Row inserted', '_id': this.lastID });
+    });
+  });
+};
+
+exports.deleteOne = (hash) => {
+  const sql = 'DELETE FROM menu WHERE hash=?';
+
+  return new Promise((resolve, reject) => {
+    db.run('DELETE FROM menu WHERE hash=?', hash, function(err) {
+      if (err)
+        reject(err);
+
+      resolve({ 'msg':'Row deleted', 'changes': this.changes });
+    });
+  });
+};
+
+exports.updateOne = (data) => {
+  const oldHash = data.hash;
+
+  const workObj = prepareData(data);
+
+  const sql = `UPDATE menu
+              SET name = ?, url = ?, md = ?, short = ?, hash = ?, meat = ?, mealtype = ?, lastused = ?
+              WHERE hash = ?`;
+
+  const newData = [workObj.name, workObj.url, workObj.md, workObj.short, workObj.hash, workObj.meat, workObj.mealtype, workObj.lastused, oldHash];
+
+  return new Promise((resolve, reject) => {
+    db.run(sql, newData, function(err) {
+      if (err)
+        reject(err);
+
+      resolve({ 'msg':'Row updated', 'changes': this.changes });
+    });
+  });
+};
+
+exports.getRandom = (timestamp) => {
+  const sql = 'SELECT _id, short, hash, name, meat, mealtype FROM menu where mealtype =1 and lastused<? order by RANDOM()';
+
+  const sqlTimestamp = ~~(timestamp / 1000);
+
+  return new Promise((resolve, reject) => {
+    db.all(sql, [sqlTimestamp], (err, rows) => {
+      if (err)
+        reject(err);
+
+      if (!err) {
+        const outgoing = [...rows];
+
+        resolve(outgoing) ;
+      }
+    });
+  });
+};
+
+exports.updateTimestamps = (newTimestamp, items) => {
+  const sqlTimestamp = ~~(newTimestamp / 1000);
+
+  console.log('>> items', items);
+  
+  const sql = 'UPDATE menu SET lastused = $lastused WHERE _id = $in';
+
+  return new Promise((resolve, reject) => {
+    db.serialize(() => {
+      const stmt = db.prepare(sql);
+
+      items.forEach((item) => {
+        console.log(item);
+        const newData = { '$lastused':sqlTimestamp, '$in':item };
+
+        stmt.run(newData);
+      });
+
+      stmt.finalize((err) => {
+        if (err)
+          reject(err);
+
+        resolve({ 'msg':'Row updateds' });
+      });
+    });
+  });
+};

server/lib/route.twitter.js

@@ -0,0 +1,13 @@
+/**
+ * Created by WebStorm.
+ * User: martin
+ * Date: 30/04/2020
+ * Time: 16:18
+
+ */
+
+const twitter = require('./twitter');
+
+module.exports = (app) => {
+  app.route('/twitter').get(twitter.all);
+};

server/lib/train.js

@@ -31,6 +31,28 @@ function getTrainTimes(req, res) {
   }
 }
 
+function getService(req, res) {
+  logger.info(`getService: ${ JSON.stringify(req.query)}`);
+
+  if (req.query.hasOwnProperty('serviceid') ) {
+    const url = `/service/${ req.query.serviceid }?accessToken=215b99fe-b237-4a01-aadc-cf315d6756d8`;
+
+    // logger.debug(`https://huxley.apphb.com${url}`);
+    // http://localhost:8100/getservice?serviceid=TDKWvQdeuviRyNYP7lk7gA
+    // "cancelReason": "This train has been cancelled because of a fault on this train",
+
+    Query(function (a, b) {
+      res.setHeader('Content-Type', 'application/json');
+
+      res.end(JSON.stringify(a));
+    }, res, 'huxley.apphb.com', url);
+  }
+  else {
+    res.setHeader('Content-Type', 'application/json');
+    res.end(JSON.stringify({}));
+  }
+}
+
 function getNextTrainTimes(req, res) {
   logger.info(`getNextTrainTimes: ${ JSON.stringify(req.query)}`);
   let trainFrom, trainTo, trainToken, url;
@@ -139,7 +161,8 @@ module.exports = {
   getTrainTimes,
   getNextTrainTimes,
   getRoute,
-  getNews
+  getNews,
+  getService
 };
 
 function toSeconds(inval) {

server/lib/twitter.js

@@ -0,0 +1,34 @@
+/**
+ * Created by WebStorm.
+ * User: martin
+ * Date: 30/04/2020
+ * Time: 16:19
+
+ */
+
+const dbmanager = require('../db/dbmanager');
+
+exports.top = (req, res) => {
+  dbmanager.getTopOne()
+    .then((data) => {
+      res.send(data);
+    })
+    .catch((err) => {
+      res.status(500).send({
+        'message': err.message || 'Some error occurred while querying the database.'
+      });
+    });
+};
+
+exports.all = (req, res) => {
+  const _list = req.body.list || [];
+  dbmanager.getAll(_list)
+    .then((data) => {
+      res.send(data);
+    })
+    .catch((err) => {
+      res.status(500).send({
+        'message': err.message || 'Some error occurred while querying the database.'
+      });
+    });
+};

test/data/file001.json

@@ -0,0 +1,91 @@
+{
+  "created_at":"Thu Apr 30 11:59:47 +0000 2020",
+  "id":1255829195940397000,
+  "id_str":"1255829195940397057",
+  "text":"@Colin56418788 @TPExpressTrains That is some serious blingage right there Colin. - Miles",
+  "display_text_range":[
+    32,
+    88
+  ],
+  "source":"<a href=\"http://www.conversocial.com\" rel=\"nofollow\">Conversocial</a>",
+  "truncated":false,
+  "in_reply_to_status_id":1255828324850466800,
+  "in_reply_to_status_id_str":"1255828324850466823",
+  "in_reply_to_user_id":1091374728168857600,
+  "in_reply_to_user_id_str":"1091374728168857600",
+  "in_reply_to_screen_name":"Colin56418788",
+  "user":{
+    "id":1143560758476906500,
+    "id_str":"1143560758476906497",
+    "name":"Avanti West Coast",
+    "screen_name":"AvantiWestCoast",
+    "location":"UK",
+    "url":"https://www.avantiwestcoast.co.uk/",
+    "description":"We run your West Coast Main Line. Here to help 24/7!",
+    "translator_type":"none",
+    "protected":false,
+    "verified":true,
+    "followers_count":33556,
+    "friends_count":24,
+    "listed_count":86,
+    "favourites_count":491,
+    "statuses_count":37934,
+    "created_at":"Tue Jun 25 16:45:05 +0000 2019",
+    "utc_offset":null,
+    "time_zone":null,
+    "geo_enabled":false,
+    "lang":null,
+    "contributors_enabled":false,
+    "is_translator":false,
+    "profile_background_color":"F5F8FA",
+    "profile_background_image_url":"",
+    "profile_background_image_url_https":"",
+    "profile_background_tile":false,
+    "profile_link_color":"1DA1F2",
+    "profile_sidebar_border_color":"C0DEED",
+    "profile_sidebar_fill_color":"DDEEF6",
+    "profile_text_color":"333333",
+    "profile_use_background_image":true,
+    "profile_image_url":"http://pbs.twimg.com/profile_images/1245999812706586624/DRpIkHjA_normal.jpg",
+    "profile_image_url_https":"https://pbs.twimg.com/profile_images/1245999812706586624/DRpIkHjA_normal.jpg",
+    "profile_banner_url":"https://pbs.twimg.com/profile_banners/1143560758476906497/1585904481",
+    "default_profile":true,
+    "default_profile_image":false,
+    "following":null,
+    "follow_request_sent":null,
+    "notifications":null
+  },
+  "geo":null,
+  "coordinates":null,
+  "place":null,
+  "contributors":null,
+  "is_quote_status":false,
+  "quote_count":0,
+  "reply_count":0,
+  "retweet_count":0,
+  "favorite_count":0,
+  "entities":{
+    "hashtags":[
+
+    ],
+    "urls":[
+
+    ],
+    "user_mentions":[
+      [
+        "Object"
+      ],
+      [
+        "Object"
+      ]
+    ],
+    "symbols":[
+
+    ]
+  },
+  "favorited":false,
+  "retweeted":false,
+  "filter_level":"low",
+  "lang":"en",
+  "timestamp_ms":"1588247987147"
+}

test/tweet.js

@@ -0,0 +1,24 @@
+/**
+ * Created by WebStorm.
+ * User: martin
+ * Date: 30/04/2020
+ * Time: 14:23
+
+ */
+const tape = require('tape');
+const _test = require('tape-promise').default; // <---- notice 'default'
+const test = _test(tape); // decorate tape
+
+const dbmanager = require('../server/db/dbmanager');
+const jsonfile = require('jsonfile');
+
+const entityPage = jsonfile.readFileSync('test/data/file001.json');
+
+test('DB tests', async function(t) {
+
+  const output = dbmanager.insertOne(entityPage);
+
+  console.log(output);
+
+  t.end();
+});

twitter.js

@@ -0,0 +1,89 @@
+/**
+ * Created by WebStorm.
+ * User: martin
+ * Date: 30/04/2020
+ * Time: 12:24
+
+ */
+
+const Twitter = require('twitter-lite');
+require('dotenv').config();
+
+const dbmanager = require('./server/db/dbmanager');
+
+
+const client = new Twitter({
+  'consumer_key' : process.env.TWITTER_CONSUMER_KEY,
+  'consumer_secret' : process.env.TWITTER_CONSUMER_SECRET,
+  'access_token_key' : process.env.TWITTER_ACCESS_TOKEN_KEY,
+  'access_token_secret' : process.env.TWITTER_ACCESS_TOKEN_SECRET
+
+});
+
+const accounts = [
+  {'name':'nationalrailenq', 'id':33546465},
+  {'name':'networkrail', 'id':365344176},
+  {'name':'NetworkRailSCOT', 'id':402687948},
+
+  {'name':'AvantiWestCoast', 'id':1143560758476906497},
+  {'name':'CalSleeper', 'id':2870293725},
+  {'name':'CrossCountryUK', 'id':153368708},
+  {'name':'Eurostar', 'id':98412169},
+  {'name':'EurostarUK', 'id':59742254},
+  {'name':'GatwickExpress', 'id':163816182},
+  {'name':'GlasgowSubway', 'id':224607925},
+  {'name':'GWRHelp', 'id':15589815},
+  {'name':'HeathrowExpress', 'id':20240678},
+  {'name':'LNER', 'id':313306238},
+  {'name':'LNRailway', 'id':910487328627535872},
+  {'name':'northernassist', 'id':194512268},
+  {'name':'ScotRail', 'id':61569136},
+  {'name':'Stansted_Exp', 'id':257511611},
+  {'name':'TfL', 'id':47319664},
+
+  {'name':'NetworkRailBHM', 'id':583910976},
+  {'name':'NetworkRailEDB', 'id':586614081},
+  {'name':'NetworkRailEUS', 'id':581807264},
+  {'name':'NetworkRailGLC', 'id':421061171},
+  {'name':'NetworkRailKGX', 'id':459192871},
+  {'name':'NetworkRailLST', 'id':581826097},
+  {'name':'NetworkRailMAN', 'id':583895871},
+  {'name':'NetworkRailVIC', 'id':587354752},
+  {'name':'BTPScotland', 'id':957256160},
+
+];
+
+
+(async function(){
+ /* const response = await user.getBearerToken();
+
+  const app = new Twitter({
+    bearer_token: response.access_token
+  });*/
+
+  const validAccounts = accounts.map(item => {
+    return item.id;
+  })
+
+
+  const parameters = {
+    follow: validAccounts.join(',') // Scotrail, networkrailScot, networkrail, nationalrail, tfl, avantiwestcoast
+  }
+
+  console.log('Starting:', parameters);
+
+  const stream = client.stream("statuses/filter", parameters)
+    .on("start", response => console.log('Started!'))
+    .on("data", tweet => {
+      console.log('>> data', tweet);
+      dbmanager.insertOne(tweet).then((v) => {
+        console.log(v);
+      }).catch((err) => {
+        console.error(err);
+      });
+    })
+    .on("ping", () => console.log('>> ping'))
+    .on("error", error => console.error(error))
+    .on("end",  response => console.log('Stopped!'));
+
+})();