martin/traintimesPWA
martin/traintimesPWA
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Twitter grabber

Browse Source 
Tweets added to server
New svelte front end
This commit is contained in:
Martin Donnelly 2020-05-04 10:34:41 +01:00
parent 4c3df0a48a
commit da2e92003f
13 changed files with 2201 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
db/init_db.js Normal file
View File

@ -0,0 +1,107 @@
print('------------------- Initialising Database Collections and Indexes --------------------------------');
/**
* Mongo shell defaults to the 'test' database unless told otherwise. So explictly choose the MDB database.
*/
db = db.getSiblingDB('TWITTER');
print('Creating collections in:');
printjson(db);
print('Existing collections:');
printjson(db.getCollectionNames());
print('Adding collections. This may take some time...');
db.createCollection('Tweets');
db.Account.dropIndexes();
db.Account.createIndex({ 'TweetID':1 }, { 'unique':true, 'sparse':true, 'name':'getTweetByID' });
/*
db.createCollection('AccountArchive');
db.AccountArchive.dropIndexes();
db.AccountArchive.createIndex({'ClientID':1},{'unique':false,'sparse':true,'name':'getAccountByClientID'});
db.createCollection('Address');
db.Address.dropIndexes();
db.Address.createIndex({'ClientID':1},{'unique':false,'sparse':true,'name':'getAddressesForClientID'});
db.Address.createIndex({'_id':1,'ClientID':1},{'unique':true,'sparse':true,'name':'getAddressForClientID'});
db.createCollection('AddressArchive');
db.AddressArchive.dropIndexes();
db.AddressArchive.createIndex({'ClientID':1},{'unique':false,'sparse':true,'name':'getAddressArchiveForClientID'});
db.createCollection('BridgeLogin');
db.BridgeLogin.dropIndexes();
db.createCollection('Client');
db.Client.dropIndexes();
db.Client.createIndex({'ClientID':1},{'unique':true,'sparse':true,'name':'getClientByClientID'});
db.Client.createIndex({'ClientName':1},{'unique':true,'sparse':true,'name':'getClientByClientName'});
db.createCollection('ClientArchive');
db.ClientArchive.dropIndexes();
db.ClientArchive.createIndex({'ClientID':1},{'unique':false,'sparse':true,'name':'getClientByClientID'});
db.createCollection('Device');
db.Device.dropIndexes();
db.Device.createIndex({'DeviceToken':1},{'unique':true,'name':'getDevicesByToken'});
db.Device.createIndex({'DeviceNumber':1},{'unique':true,'sparse':true,'name':'getDevicesByNumber'});
db.Device.createIndex({'ClientID':1},{'unique':false,'name':'getDevicesByClientID'});
db.createCollection('DeviceArchive');
db.DeviceArchive.dropIndexes();
db.DeviceArchive.createIndex({'ClientID':1},{'unique':false,'name':'getDevicesByClientID'});
db.createCollection('Images');
db.Images.dropIndexes();
db.Images.createIndex({'ClientID':1},{'unique':false,'sparse':true,'name':'getImagesByClientID'});
db.createCollection('Items');
db.Items.dropIndexes();
db.Items.createIndex({'ClientID':1},{'unique':false,'sparse':true,'name':'getItemsByClientID'});
db.createCollection('Messages');
db.Messages.dropIndexes();
db.Messages.createIndex({'ClientID':1,'TimeFilter':1},{'unique':false,'sparse':true,'name':'getMessagesByClientIDandTime'});
db.createCollection('MessagesArchive');
db.MessagesArchive.dropIndexes();
db.MessagesArchive.createIndex({'ClientID':1},{'unique':false,'sparse':true,'name':'getMessagesByClientID'});
db.createCollection('PayCode');
db.PayCode.dropIndexes();
db.PayCode.createIndex({'PayCode':1},{'unique':true,'sparse':true,'name':'getCurrentPayCodes'});
db.PayCode.createIndex({'Expiry':1},{expireAfterSeconds:0,'name':'payCodeTTL'});
db.createCollection('SystemLog');
db.SystemLog.dropIndexes();
db.SystemLog.createIndex({'DateTime':-1},{'unique':false,'sparse':true,'name':'getEntryByTime'});
db.createCollection('Transaction');
db.Transaction.dropIndexes();
db.Transaction.createIndex({'MerchantInvoiceNumber.InvoiceNumber':-1,'MerchantInvoiceNumber.MerchantID':-1,'MerchantInvoiceNumber.MerchantIndex':1},{'unique':true,partialFilterExpression:{MerchantInvoiceNumber:{$exists:true}},name:'MerchantInvoiceNumberIndex'});
db.createCollection('TransactionArchive');
db.TransactionArchive.dropIndexes();
db.TransactionArchive.createIndex({'CustomerClientID':1},{'unique':false,'sparse':true,'name':'getTransactionsByClientID'});
db.createCollection('TransactionHistory');
db.TransactionHistory.dropIndexes();
db.TransactionHistory.createIndex({'ClientID':1,'AccountID':1,'SaleTime':1},{'unique':false,'sparse':true,'name':'getHistoryByClient'});
db.TransactionHistory.createIndex({'AccountID':1},{'unique':false,'sparse':true,'name':'getHistoryByAccount'});
db.TransactionHistory.createIndex({'OtherImage':1},{'unique':false,'sparse':true,'name':'findImage'});
db.createCollection('TwoFARequests');
db.TwoFARequests.dropIndexes();
db.TwoFARequests.createIndex({'RequestExpiry':1},{'unique':false,expireAfterSeconds:0,'name':'requestTTL'});
db.createCollection('WebConsoleSessions');
db.WebConsoleSessions.dropIndexes();
db.WebConsoleSessions.createIndex({'expires': 1}, {expireAfterSeconds: 60, 'name': 'webConsoleSessionsTTL'});
*/
print('Collections added.');
print('Updated Collections:');
printjson(db.getCollectionNames());
print('------------------- DONE Initialising Database --------------------------------');

BIN
db/twitter.db Normal file
View File

Binary file not shown.

1683
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

6
package.json
View File

@ -16,14 +16,18 @@
"browserify": "^14.5.0",
"compressjs": "^1.0.3",
"cors": "^2.8.4",
"dotenv": "^8.2.0",
"es6-promise": "^4.1.1",
"express": "^4.16.2",
"fecha": "^4.2.0",
"jquery": "^3.2.1",
"json-lzw": "^0.3.2",
"log4js": "^2.4.1",
"lz77": "^1.1.0",
"minibus": "^3.1.0",
"muicss": "^0.9.33",
"sqlite3": "^4.2.0",
"twitter-lite": "^0.11.0",
"underscore": "^1.8.3"
},
"devDependencies": {
@ -66,6 +70,8 @@
"mocha": "^3.5.3",
"require-dir": "^0.3.2",
"sinon": "^4.1.1",
"tape": "^5.0.0",
"tape-promise": "^4.0.0",
"vinyl-buffer": "^1.0.0",
"vinyl-source-stream": "^1.1.0",
"watchify": "^3.9.0"

5
server.js
View File

@ -6,6 +6,7 @@ const apicache = require('apicache');
const logger = require('log4js').getLogger('Server');
const train = require('./server/lib/train');
const twitter = require('./server/lib/twitter');
logger.level = 'debug';
const app = express();
@ -15,6 +16,7 @@ const sitePath = 'live';
apicache.options({ 'debug': true });
const cache = apicache.middleware;
app.use(express.json());
app.use(express.static(path.join(__dirname, sitePath)));
@ -24,6 +26,9 @@ app.use('/gettrains', cors(), cache('60 seconds'), train.getTrainTimes);
app.use('/getnexttraintimes', cors(), cache('60 seconds'), train.getNextTrainTimes);
app.use('/getroute', cors(), cache('60 seconds'), train.getRoute);
app.use('/getnews', cors(), cache('30 minutes'), train.getNews);
app.use('/getservice', cors(), cache('60 seconds'), train.getService);
app.use('/twitter', cors(), twitter.all);
process.on('uncaughtException', err => {
logger.error(err);

5
server/db/connect.js Normal file
View File

@ -0,0 +1,5 @@
const sqlite3 = require('sqlite3').verbose();
console.log(`${__dirname}/../../db/twitter.db`);
const db = new sqlite3.Database(`${__dirname}/../../db/twitter.db`);
module.exports = db;

180
server/db/dbmanager.js Normal file
View File

@ -0,0 +1,180 @@
const db = require('./connect');
const fecha = require('fecha');
// exports.create = (req, res) => {
function prepareData(_obj) {
const newObj = Object.assign({}, { 'tweet':JSON.stringify(_obj) });
newObj.ts = parseInt(_obj.timestamp_ms, 10);
newObj.createdAt = fecha.parse(_obj.created_at, 'ddd MMM DD HH:mm:ss ZZ YYYY').getTime(); // Thu Apr 30 11:59:47 +0000 2020
newObj.id = _obj.id;
newObj.idStr = _obj.id_str;
newObj.userID = _obj.user.id;
newObj.userIDStr = _obj.user.id_str;
newObj.name = _obj.user.name;
newObj.screenName = _obj.user.screen_name;
return newObj;
}
exports.prepareData = prepareData;
exports.getAll = (list) => {
console.log('>> getAll:', list);
const outgoing = [];
// const sql = 'SELECT * FROM tweets order by _id desc limit 20';
const sql = `SELECT * FROM tweets where userID In ( ${list.join(',')} ) order by _id desc limit 20`;
return new Promise((resolve, reject) => {
db.all(sql, [], (err, rows) => {
if (err)
reject(err);
rows.forEach((row) => {
outgoing.push(row);
});
resolve(outgoing) ;
});
});
};
exports.getTopOne = (hash) => {
const sql = 'SELECT * FROM tweets order by _id desc limit 1';
return new Promise((resolve, reject) => {
db.get(sql, [hash], (err, row) => {
if (err)
reject(err);
if (!err) resolve([row]);
});
});
};
exports.getOne = (hash) => {
const sql = 'SELECT * FROM menu WHERE hash = ?';
return new Promise((resolve, reject) => {
db.get(sql, [hash], (err, row) => {
if (err)
reject(err);
if (!err) resolve(row);
});
});
};
exports.getOneShort = (short) => {
const sql = 'SELECT * FROM menu WHERE short = ?';
return new Promise((resolve, reject) => {
db.get(sql, [short], (err, row) => {
if (err)
reject(err);
if (!err) resolve(row);
});
});
};
exports.insertOne = (data) => {
const sql = 'INSERT INTO tweets VALUES (?,?,?,?,?,?,?,?,?,?)';
const tweetObj = prepareData(data);
return new Promise((resolve, reject) => {
db.run(sql, [null, tweetObj.ts, tweetObj.createdAt, tweetObj.id, tweetObj.idStr, tweetObj.userID, tweetObj.userIDStr, tweetObj.name, tweetObj.screenName, tweetObj.tweet], function(err) {
if (err)
reject(err);
resolve({ 'msg':'Row inserted', '_id': this.lastID });
});
});
};
exports.deleteOne = (hash) => {
const sql = 'DELETE FROM menu WHERE hash=?';
return new Promise((resolve, reject) => {
db.run('DELETE FROM menu WHERE hash=?', hash, function(err) {
if (err)
reject(err);
resolve({ 'msg':'Row deleted', 'changes': this.changes });
});
});
};
exports.updateOne = (data) => {
const oldHash = data.hash;
const workObj = prepareData(data);
const sql = `UPDATE menu
SET name = ?, url = ?, md = ?, short = ?, hash = ?, meat = ?, mealtype = ?, lastused = ?
WHERE hash = ?`;
const newData = [workObj.name, workObj.url, workObj.md, workObj.short, workObj.hash, workObj.meat, workObj.mealtype, workObj.lastused, oldHash];
return new Promise((resolve, reject) => {
db.run(sql, newData, function(err) {
if (err)
reject(err);
resolve({ 'msg':'Row updated', 'changes': this.changes });
});
});
};
exports.getRandom = (timestamp) => {
const sql = 'SELECT _id, short, hash, name, meat, mealtype FROM menu where mealtype =1 and lastused<? order by RANDOM()';
const sqlTimestamp = ~~(timestamp / 1000);
return new Promise((resolve, reject) => {
db.all(sql, [sqlTimestamp], (err, rows) => {
if (err)
reject(err);
if (!err) {
const outgoing = [...rows];
resolve(outgoing) ;
}
});
});
};
exports.updateTimestamps = (newTimestamp, items) => {
const sqlTimestamp = ~~(newTimestamp / 1000);
console.log('>> items', items);
const sql = 'UPDATE menu SET lastused = $lastused WHERE _id = $in';
return new Promise((resolve, reject) => {
db.serialize(() => {
const stmt = db.prepare(sql);
items.forEach((item) => {
console.log(item);
const newData = { '$lastused':sqlTimestamp, '$in':item };
stmt.run(newData);
});
stmt.finalize((err) => {
if (err)
reject(err);
resolve({ 'msg':'Row updateds' });
});
});
});
};

13
server/lib/route.twitter.js Normal file
View File

@ -0,0 +1,13 @@
/**
* Created by WebStorm.
* User: martin
* Date: 30/04/2020
* Time: 16:18
*/
const twitter = require('./twitter');
module.exports = (app) => {
app.route('/twitter').get(twitter.all);
};

25
server/lib/train.js
View File

@ -31,6 +31,28 @@ function getTrainTimes(req, res) {
}
}
function getService(req, res) {
logger.info(`getService: ${ JSON.stringify(req.query)}`);
if (req.query.hasOwnProperty('serviceid') ) {
const url = `/service/${ req.query.serviceid }?accessToken=215b99fe-b237-4a01-aadc-cf315d6756d8`;
// logger.debug(`https://huxley.apphb.com${url}`);
// http://localhost:8100/getservice?serviceid=TDKWvQdeuviRyNYP7lk7gA
// "cancelReason": "This train has been cancelled because of a fault on this train",
Query(function (a, b) {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify(a));
}, res, 'huxley.apphb.com', url);
}
else {
res.setHeader('Content-Type', 'application/json');
res.end(JSON.stringify({}));
}
}
function getNextTrainTimes(req, res) {
logger.info(`getNextTrainTimes: ${ JSON.stringify(req.query)}`);
let trainFrom, trainTo, trainToken, url;
@ -139,7 +161,8 @@ module.exports = {
getTrainTimes,
getNextTrainTimes,
getRoute,
getNews
getNews,
getService
};
function toSeconds(inval) {

34
server/lib/twitter.js Normal file
View File

@ -0,0 +1,34 @@
/**
* Created by WebStorm.
* User: martin
* Date: 30/04/2020
* Time: 16:19
*/
const dbmanager = require('../db/dbmanager');
exports.top = (req, res) => {
dbmanager.getTopOne()
.then((data) => {
res.send(data);
})
.catch((err) => {
res.status(500).send({
'message': err.message || 'Some error occurred while querying the database.'
});
});
};
exports.all = (req, res) => {
const _list = req.body.list || [];
dbmanager.getAll(_list)
.then((data) => {
res.send(data);
})
.catch((err) => {
res.status(500).send({
'message': err.message || 'Some error occurred while querying the database.'
});
});
};

91
test/data/file001.json Normal file
View File

@ -0,0 +1,91 @@
{
"created_at":"Thu Apr 30 11:59:47 +0000 2020",
"id":1255829195940397000,
"id_str":"1255829195940397057",
"text":"@Colin56418788 @TPExpressTrains That is some serious blingage right there Colin. - Miles",
"display_text_range":[
32,
88
],
"source":"<a href=\"http://www.conversocial.com\" rel=\"nofollow\">Conversocial</a>",
"truncated":false,
"in_reply_to_status_id":1255828324850466800,
"in_reply_to_status_id_str":"1255828324850466823",
"in_reply_to_user_id":1091374728168857600,
"in_reply_to_user_id_str":"1091374728168857600",
"in_reply_to_screen_name":"Colin56418788",
"user":{
"id":1143560758476906500,
"id_str":"1143560758476906497",
"name":"Avanti West Coast",
"screen_name":"AvantiWestCoast",
"location":"UK",
"url":"https://www.avantiwestcoast.co.uk/",
"description":"We run your West Coast Main Line. Here to help 24/7!",
"translator_type":"none",
"protected":false,
"verified":true,
"followers_count":33556,
"friends_count":24,
"listed_count":86,
"favourites_count":491,
"statuses_count":37934,
"created_at":"Tue Jun 25 16:45:05 +0000 2019",
"utc_offset":null,
"time_zone":null,
"geo_enabled":false,
"lang":null,
"contributors_enabled":false,
"is_translator":false,
"profile_background_color":"F5F8FA",
"profile_background_image_url":"",
"profile_background_image_url_https":"",
"profile_background_tile":false,
"profile_link_color":"1DA1F2",
"profile_sidebar_border_color":"C0DEED",
"profile_sidebar_fill_color":"DDEEF6",
"profile_text_color":"333333",
"profile_use_background_image":true,
"profile_image_url":"http://pbs.twimg.com/profile_images/1245999812706586624/DRpIkHjA_normal.jpg",
"profile_image_url_https":"https://pbs.twimg.com/profile_images/1245999812706586624/DRpIkHjA_normal.jpg",
"profile_banner_url":"https://pbs.twimg.com/profile_banners/1143560758476906497/1585904481",
"default_profile":true,
"default_profile_image":false,
"following":null,
"follow_request_sent":null,
"notifications":null
},
"geo":null,
"coordinates":null,
"place":null,
"contributors":null,
"is_quote_status":false,
"quote_count":0,
"reply_count":0,
"retweet_count":0,
"favorite_count":0,
"entities":{
"hashtags":[
],
"urls":[
],
"user_mentions":[
[
"Object"
],
[
"Object"
]
],
"symbols":[
]
},
"favorited":false,
"retweeted":false,
"filter_level":"low",
"lang":"en",
"timestamp_ms":"1588247987147"
}

24
test/tweet.js Normal file
View File

@ -0,0 +1,24 @@
/**
* Created by WebStorm.
* User: martin
* Date: 30/04/2020
* Time: 14:23
*/
const tape = require('tape');
const _test = require('tape-promise').default; // <---- notice 'default'
const test = _test(tape); // decorate tape
const dbmanager = require('../server/db/dbmanager');
const jsonfile = require('jsonfile');
const entityPage = jsonfile.readFileSync('test/data/file001.json');
test('DB tests', async function(t) {
const output = dbmanager.insertOne(entityPage);
console.log(output);
t.end();
});

89
twitter.js Normal file
View File

@ -0,0 +1,89 @@
/**
* Created by WebStorm.
* User: martin
* Date: 30/04/2020
* Time: 12:24
*/
const Twitter = require('twitter-lite');
require('dotenv').config();
const dbmanager = require('./server/db/dbmanager');
const client = new Twitter({
'consumer_key' : process.env.TWITTER_CONSUMER_KEY,
'consumer_secret' : process.env.TWITTER_CONSUMER_SECRET,
'access_token_key' : process.env.TWITTER_ACCESS_TOKEN_KEY,
'access_token_secret' : process.env.TWITTER_ACCESS_TOKEN_SECRET
});
const accounts = [
{'name':'nationalrailenq', 'id':33546465},
{'name':'networkrail', 'id':365344176},
{'name':'NetworkRailSCOT', 'id':402687948},
{'name':'AvantiWestCoast', 'id':1143560758476906497},
{'name':'CalSleeper', 'id':2870293725},
{'name':'CrossCountryUK', 'id':153368708},
{'name':'Eurostar', 'id':98412169},
{'name':'EurostarUK', 'id':59742254},
{'name':'GatwickExpress', 'id':163816182},
{'name':'GlasgowSubway', 'id':224607925},
{'name':'GWRHelp', 'id':15589815},
{'name':'HeathrowExpress', 'id':20240678},
{'name':'LNER', 'id':313306238},
{'name':'LNRailway', 'id':910487328627535872},
{'name':'northernassist', 'id':194512268},
{'name':'ScotRail', 'id':61569136},
{'name':'Stansted_Exp', 'id':257511611},
{'name':'TfL', 'id':47319664},
{'name':'NetworkRailBHM', 'id':583910976},
{'name':'NetworkRailEDB', 'id':586614081},
{'name':'NetworkRailEUS', 'id':581807264},
{'name':'NetworkRailGLC', 'id':421061171},
{'name':'NetworkRailKGX', 'id':459192871},
{'name':'NetworkRailLST', 'id':581826097},
{'name':'NetworkRailMAN', 'id':583895871},
{'name':'NetworkRailVIC', 'id':587354752},
{'name':'BTPScotland', 'id':957256160},
];
(async function(){
/* const response = await user.getBearerToken();
const app = new Twitter({
bearer_token: response.access_token
});*/
const validAccounts = accounts.map(item => {
return item.id;
})
const parameters = {
follow: validAccounts.join(',') // Scotrail, networkrailScot, networkrail, nationalrail, tfl, avantiwestcoast
}
console.log('Starting:', parameters);
const stream = client.stream("statuses/filter", parameters)
.on("start", response => console.log('Started!'))
.on("data", tweet => {
console.log('>> data', tweet);
dbmanager.insertOne(tweet).then((v) => {
console.log(v);
}).catch((err) => {
console.error(err);
});
})
.on("ping", () => console.log('>> ping'))
.on("error", error => console.error(error))
.on("end", response => console.log('Stopped!'));
})();