martin/jubilee
martin/jubilee
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fixed news gathering again

Browse Source
This commit is contained in:
Martin Donnelly 2020-05-10 11:59:35 +01:00
parent 762a0cf519
commit a57fc96bd3
11 changed files with 6165 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

113
package-lock.json generated
View File

@ -170,6 +170,7 @@
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz",
"integrity": "sha512-JVwXMr9nHYTUXsBFKUqhJwvlcYU/blreOEUkhNR2eXZIvwd+c+o5V4MgDPKWnMS/56awN3TRzIP+KoPn+roQtg==",
"optional": true,
"requires": {
"es6-promisify": "^5.0.0"
}
@ -904,6 +905,7 @@
"version": "2.10.1",
"resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz",
"integrity": "sha1-OciRjO/1eZ+D+UkqhI9iWt0Mdm8=",
"optional": true,
"requires": {
"hoek": "2.x.x"
}
@ -1153,7 +1155,8 @@
"buffer-more-ints": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-1.0.0.tgz",
"integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg=="
"integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==",
"optional": true
},
"buffer-shims": {
"version": "1.0.0",
@ -1714,6 +1717,15 @@
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
},
"cors": {
"version": "2.8.5",
"resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
"integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==",
"requires": {
"object-assign": "^4",
"vary": "^1"
}
},
"create-ecdh": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.3.tgz",
@ -2534,6 +2546,7 @@
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz",
"integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=",
"optional": true,
"requires": {
"es6-promise": "^4.0.3"
},
@ -2541,7 +2554,8 @@
"es6-promise": {
"version": "4.2.5",
"resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.5.tgz",
"integrity": "sha512-n6wvpdE43VFtJq+lUDYDBFUwV8TZbuGXLV4D6wKafg13ldznKsyEvatubnmUe31zcvelSzOHF+XbaT+Bl9ObDg=="
"integrity": "sha512-n6wvpdE43VFtJq+lUDYDBFUwV8TZbuGXLV4D6wKafg13ldznKsyEvatubnmUe31zcvelSzOHF+XbaT+Bl9ObDg==",
"optional": true
}
}
},
@ -2754,7 +2768,8 @@
"esprima": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/esprima/-/esprima-3.1.3.tgz",
"integrity": "sha1-/cpRzuYTOJXjyI1TXOSdv/YqRjM="
"integrity": "sha1-/cpRzuYTOJXjyI1TXOSdv/YqRjM=",
"optional": true
},
"esquery": {
"version": "1.0.1",
@ -3429,7 +3444,8 @@
"ansi-regex": {
"version": "2.1.1",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"aproba": {
"version": "1.2.0",
@ -3450,12 +3466,14 @@
"balanced-match": {
"version": "1.0.0",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"brace-expansion": {
"version": "1.1.11",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
@ -3470,17 +3488,20 @@
"code-point-at": {
"version": "1.1.0",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"concat-map": {
"version": "0.0.1",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"console-control-strings": {
"version": "1.1.0",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"core-util-is": {
"version": "1.0.2",
@ -3597,7 +3618,8 @@
"inherits": {
"version": "2.0.3",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"ini": {
"version": "1.3.5",
@ -3609,6 +3631,7 @@
"version": "1.0.0",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"number-is-nan": "^1.0.0"
}
@ -3623,6 +3646,7 @@
"version": "3.0.4",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"brace-expansion": "^1.1.7"
}
@ -3630,12 +3654,14 @@
"minimist": {
"version": "0.0.8",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"minipass": {
"version": "2.2.4",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"safe-buffer": "^5.1.1",
"yallist": "^3.0.0"
@ -3654,6 +3680,7 @@
"version": "0.5.1",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"minimist": "0.0.8"
}
@ -3741,7 +3768,8 @@
"number-is-nan": {
"version": "1.0.1",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"object-assign": {
"version": "4.1.1",
@ -3753,6 +3781,7 @@
"version": "1.4.0",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"wrappy": "1"
}
@ -3838,7 +3867,8 @@
"safe-buffer": {
"version": "5.1.1",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"safer-buffer": {
"version": "2.1.2",
@ -3874,6 +3904,7 @@
"version": "1.0.2",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"code-point-at": "^1.0.0",
"is-fullwidth-code-point": "^1.0.0",
@ -3893,6 +3924,7 @@
"version": "3.0.1",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"ansi-regex": "^2.0.0"
}
@ -3936,12 +3968,14 @@
"wrappy": {
"version": "1.0.2",
"bundled": true,
"dev": true
"dev": true,
"optional": true
},
"yallist": {
"version": "3.0.2",
"bundled": true,
"dev": true
"dev": true,
"optional": true
}
}
},
@ -5341,7 +5375,8 @@
"hoek": {
"version": "2.16.3",
"resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz",
"integrity": "sha1-ILt0A9POo5jpHcRxCo/xuCdKJe0="
"integrity": "sha1-ILt0A9POo5jpHcRxCo/xuCdKJe0=",
"optional": true
},
"homedir-polyfill": {
"version": "1.0.1",
@ -5438,6 +5473,7 @@
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-2.1.0.tgz",
"integrity": "sha512-qwHbBLV7WviBl0rQsOzH6o5lwyOIvwp/BdFnvVxXORldu5TmjFfjzBcWUWS5kWAZhmv+JtiDhSuQCp4sBfbIgg==",
"optional": true,
"requires": {
"agent-base": "4",
"debug": "3.1.0"
@ -5447,6 +5483,7 @@
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
"integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==",
"optional": true,
"requires": {
"ms": "2.0.0"
}
@ -5467,6 +5504,7 @@
"version": "1.6.1",
"resolved": "https://registry.npmjs.org/httpntlm/-/httpntlm-1.6.1.tgz",
"integrity": "sha1-rQFScUOi6Hc8+uapb1hla7UqNLI=",
"optional": true,
"requires": {
"httpreq": ">=0.4.22",
"underscore": "~1.7.0"
@ -5475,14 +5513,16 @@
"underscore": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz",
"integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk="
"integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk=",
"optional": true
}
}
},
"httpreq": {
"version": "0.4.24",
"resolved": "https://registry.npmjs.org/httpreq/-/httpreq-0.4.24.tgz",
"integrity": "sha1-QzX/2CzZaWaKOUZckprGHWOTYn8="
"integrity": "sha1-QzX/2CzZaWaKOUZckprGHWOTYn8=",
"optional": true
},
"https-browserify": {
"version": "1.0.0",
@ -5493,6 +5533,7 @@
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz",
"integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==",
"optional": true,
"requires": {
"agent-base": "^4.1.0",
"debug": "^3.1.0"
@ -5502,6 +5543,7 @@
"version": "3.2.6",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
"integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
"optional": true,
"requires": {
"ms": "^2.1.1"
}
@ -5509,7 +5551,8 @@
"ms": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz",
"integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg=="
"integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==",
"optional": true
}
}
},
@ -5717,7 +5760,8 @@
"ip": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz",
"integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo="
"integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=",
"optional": true
},
"ipaddr.js": {
"version": "1.8.0",
@ -5982,7 +6026,8 @@
"is-property": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz",
"integrity": "sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ="
"integrity": "sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ=",
"optional": true
},
"is-regex": {
"version": "1.0.4",
@ -6257,12 +6302,14 @@
"libbase64": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/libbase64/-/libbase64-0.1.0.tgz",
"integrity": "sha1-YjUag5VjrF/1vSbxL2Dpgwu3UeY="
"integrity": "sha1-YjUag5VjrF/1vSbxL2Dpgwu3UeY=",
"optional": true
},
"libmime": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/libmime/-/libmime-3.0.0.tgz",
"integrity": "sha1-UaGp50SOy9Ms2lRCFnW7IbwJPaY=",
"optional": true,
"requires": {
"iconv-lite": "0.4.15",
"libbase64": "0.1.0",
@ -6272,14 +6319,16 @@
"iconv-lite": {
"version": "0.4.15",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.15.tgz",
"integrity": "sha1-/iZaIYrGpXz+hUkn6dBMGYJe3es="
"integrity": "sha1-/iZaIYrGpXz+hUkn6dBMGYJe3es=",
"optional": true
}
}
},
"libqp": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/libqp/-/libqp-1.1.0.tgz",
"integrity": "sha1-9ebgatdLeU+1tbZpiL9yjvHe2+g="
"integrity": "sha1-9ebgatdLeU+1tbZpiL9yjvHe2+g=",
"optional": true
},
"liftoff": {
"version": "2.5.0",
@ -7499,12 +7548,14 @@
"nodemailer-fetch": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/nodemailer-fetch/-/nodemailer-fetch-1.6.0.tgz",
"integrity": "sha1-ecSQihwPXzdbc/6IjamCj23JY6Q="
"integrity": "sha1-ecSQihwPXzdbc/6IjamCj23JY6Q=",
"optional": true
},
"nodemailer-shared": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/nodemailer-shared/-/nodemailer-shared-1.1.0.tgz",
"integrity": "sha1-z1mU4v0mjQD1zw+nZ6CBae2wfsA=",
"optional": true,
"requires": {
"nodemailer-fetch": "1.6.0"
}
@ -7534,7 +7585,8 @@
"nodemailer-wellknown": {
"version": "0.1.10",
"resolved": "https://registry.npmjs.org/nodemailer-wellknown/-/nodemailer-wellknown-0.1.10.tgz",
"integrity": "sha1-WG24EB2zDLRDjrVGc3pBqtDPE9U="
"integrity": "sha1-WG24EB2zDLRDjrVGc3pBqtDPE9U=",
"optional": true
},
"nopt": {
"version": "3.0.6",
@ -9457,6 +9509,11 @@
"resolve": "^1.1.6"
}
},
"redaxios": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/redaxios/-/redaxios-0.2.0.tgz",
"integrity": "sha512-lrDld2bVWIBrW+S1HPMyP8OvEtpHPIfmXyW7T7xTox1SM4/M3aC5X55xa2vRR3LZHwCvpMQAgCye8IyICzkKpA=="
},
"redent": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/redent/-/redent-1.0.0.tgz",
@ -10260,12 +10317,14 @@
"smart-buffer": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.0.2.tgz",
"integrity": "sha512-JDhEpTKzXusOqXZ0BUIdH+CjFdO/CR3tLlf5CN34IypI+xMmXW1uB16OOY8z3cICbJlDAVJzNbwBhNO0wt9OAw=="
"integrity": "sha512-JDhEpTKzXusOqXZ0BUIdH+CjFdO/CR3tLlf5CN34IypI+xMmXW1uB16OOY8z3cICbJlDAVJzNbwBhNO0wt9OAw==",
"optional": true
},
"smtp-connection": {
"version": "2.12.0",
"resolved": "https://registry.npmjs.org/smtp-connection/-/smtp-connection-2.12.0.tgz",
"integrity": "sha1-1275EnyyPCJZ7bHoNJwujV4tdME=",
"optional": true,
"requires": {
"httpntlm": "1.6.1",
"nodemailer-shared": "1.1.0"
@ -10391,6 +10450,7 @@
"version": "2.2.3",
"resolved": "https://registry.npmjs.org/socks/-/socks-2.2.3.tgz",
"integrity": "sha512-+2r83WaRT3PXYoO/1z+RDEBE7Z2f9YcdQnJ0K/ncXXbV5gJ6wYfNAebYFYiiUjM6E4JyXnPY8cimwyvFYHVUUA==",
"optional": true,
"requires": {
"ip": "^1.1.5",
"smart-buffer": "4.0.2"
@ -10400,6 +10460,7 @@
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-4.0.1.tgz",
"integrity": "sha512-Kezx6/VBguXOsEe5oU3lXYyKMi4+gva72TwJ7pQY5JfqUx2nMk7NXA6z/mpNqIlfQjWYVfeuNvQjexiTaTn6Nw==",
"optional": true,
"requires": {
"agent-base": "~4.2.0",
"socks": "~2.2.0"

4
package.json
View File

@ -7,7 +7,7 @@
"test": "mocha"
},
"engines": {
"node": ">=8.9.1"
"node": "=8.17.0"
},
"author": "",
"license": "ISC",
@ -16,6 +16,7 @@
"backbone": "^1.3.3",
"browserify": "^16.2.3",
"cheerio": "^1.0.0-rc.2",
"cors": "^2.8.5",
"dark-sky-api": "^0.6.32",
"dateformat": "^3.0.3",
"debug-logger": "^0.4.1",
@ -49,6 +50,7 @@
"muicss": "^0.9.41",
"node-foursquare-venues": "^1.1.0",
"openweather-apis": "^3.3.5",
"redaxios": "^0.2.0",
"request-json": "^0.6.4",
"request-promise-native": "^1.0.5",
"strict-uri-encode": "^2.0.0",

5
server.js
View File

@ -10,9 +10,14 @@ const agenda = require('./server/agenda');
const directions = require('./server/directions');
const geocode = require('./server/geocode');
const cors = require('cors');
logger.level = 'debug';
const app = express();
app.use(cors());
const port = process.env.PORT || 8110;
const sitePath = 'live';

17
server/reducers/euronews.js
View File

@ -20,7 +20,13 @@ function reduceArticle(body = '') {
const html = [];
const content = $('div.article__content');
const content = $('div.c-article-content');
const ampAddStripper = /<\s*amp-ad(\s+.*?>|>).*?<\s*\/\s*amp-ad\s*>/ig;
$(content).find('amp-ad').remove();
$(content).find('div.widget__wrapper').remove();
$(content).find('div.widget').remove();
for (let top = 0, topLen = content.length; top < topLen; top++) {
const children = $(content[top]).children();
@ -31,6 +37,8 @@ function reduceArticle(body = '') {
const symbol = /src=(['"])(http[s]?:\/\/)/.exec(line) || [];
if (tag === 'amp-twitter') {
const tweetid = $(children[index]).data('tweetid');
line = `<amp-twitter width="375"
@ -76,7 +84,12 @@ function reduceArticleV2(body = '') {
const html = [];
const content = $('div.article__content');
const content = $('div.c-article-content');
const ampAddStripper = /<\s*amp-ad(\s+.*?>|>).*?<\s*\/\s*amp-ad\s*>/ig;
$(content).find('amp-ad').remove();
$(content).find('div.widget__wrapper').remove();
for (let top = 0, topLen = content.length; top < topLen; top++) {
const children = $(content[top]).children();

4
src/v1/js/Weather.js
View File

@ -124,10 +124,10 @@ const WeatherModel = Backbone.Model.extend({
}
}.bind(this));
},
'logUpdate': function() {
'logUpdate': function () {
console.log('Weather logging:');
const log = { 'lat' : this.get('latitude'), 'long': this.get('longitude'), 'time': new Date().getTime() };
const log = { 'lat': this.get('latitude'), 'long': this.get('longitude'), 'time': new Date().getTime() };
this.set('log', log, { 'silent': true });

965
test/data/euronews/20200506-widgets.html Normal file
View File

File diff suppressed because one or more lines are too long

2263
test/data/euronews/20200506ampRedirectedToHtml.html Normal file
View File

File diff suppressed because one or more lines are too long

925
test/data/euronews/20200506chinaspace.html Normal file
View File

File diff suppressed because one or more lines are too long

957
test/data/euronews/20200506impact.html Normal file
View File

File diff suppressed because one or more lines are too long

934
test/data/euronews/20200506lifelockdown.html Normal file
View File

File diff suppressed because one or more lines are too long

12
test/euronews.scrape.js
View File

@ -7,14 +7,14 @@ const jsonfile = require('jsonfile');
const cheerio = require('cheerio');
const { reduceArticle,reduceArticleV2 } = require('../server/reducers/euronews');
const { reduceArticle, reduceArticleV2 } = require('../server/reducers/euronews');
const psDetail = fs.readFileSync('test/data/euronews/20200506-widgets.html');
test('Euronews', async t => {
t.skip('Reduce a page', async t => {
const psDetail = fs.readFileSync('./data/euronews/amp-20200114.html');
t.test('Reduce a page', async t => {
const expectedJSON = { 'bob':false }; // jsonfile.readFileSync('tests/data/cz/ps001.json');
const output = await reduceArticle(psDetail);
t.deepEquals(output, expectedJSON, 'Extracted Details from Page');
@ -22,12 +22,9 @@ test('Euronews', async t => {
t.end();
});
t.test('Reduce a page V2', async t => {
const psDetail = fs.readFileSync('./data/euronews/amp-20200114.html');
const expectedJSON = { 'bob':false }; // jsonfile.readFileSync('tests/data/cz/ps001.json');
const output = await reduceArticleV2(psDetail);
t.deepEquals(output, expectedJSON, 'Extracted Details from Page');
@ -35,6 +32,5 @@ test('Euronews', async t => {
t.end();
});
t.end();
});