From a57fc96bd347948aa4671e8b7cd37d63ad514d70 Mon Sep 17 00:00:00 2001 From: Martin Donnelly Date: Sun, 10 May 2020 11:59:35 +0100 Subject: [PATCH] fixed news gathering again --- package-lock.json | 113 +- package.json | 4 +- server.js | 5 + server/reducers/euronews.js | 17 +- src/v1/js/Weather.js | 4 +- test/data/euronews/20200506-widgets.html | 965 +++++++ .../euronews/20200506ampRedirectedToHtml.html | 2263 +++++++++++++++++ test/data/euronews/20200506chinaspace.html | 925 +++++++ test/data/euronews/20200506impact.html | 957 +++++++ test/data/euronews/20200506lifelockdown.html | 934 +++++++ test/euronews.scrape.js | 22 +- 11 files changed, 6165 insertions(+), 44 deletions(-) create mode 100644 test/data/euronews/20200506-widgets.html create mode 100644 test/data/euronews/20200506ampRedirectedToHtml.html create mode 100644 test/data/euronews/20200506chinaspace.html create mode 100644 test/data/euronews/20200506impact.html create mode 100644 test/data/euronews/20200506lifelockdown.html diff --git a/package-lock.json b/package-lock.json index fcca00f..3c3acb5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -170,6 +170,7 @@ "version": "4.2.1", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.1.tgz", "integrity": "sha512-JVwXMr9nHYTUXsBFKUqhJwvlcYU/blreOEUkhNR2eXZIvwd+c+o5V4MgDPKWnMS/56awN3TRzIP+KoPn+roQtg==", + "optional": true, "requires": { "es6-promisify": "^5.0.0" } @@ -904,6 +905,7 @@ "version": "2.10.1", "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz", "integrity": "sha1-OciRjO/1eZ+D+UkqhI9iWt0Mdm8=", + "optional": true, "requires": { "hoek": "2.x.x" } @@ -1153,7 +1155,8 @@ "buffer-more-ints": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-1.0.0.tgz", - "integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==" + "integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==", + "optional": true }, "buffer-shims": { "version": "1.0.0", @@ -1714,6 +1717,15 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, + "cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "requires": { + "object-assign": "^4", + "vary": "^1" + } + }, "create-ecdh": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.3.tgz", @@ -2534,6 +2546,7 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", + "optional": true, "requires": { "es6-promise": "^4.0.3" }, @@ -2541,7 +2554,8 @@ "es6-promise": { "version": "4.2.5", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.5.tgz", - "integrity": "sha512-n6wvpdE43VFtJq+lUDYDBFUwV8TZbuGXLV4D6wKafg13ldznKsyEvatubnmUe31zcvelSzOHF+XbaT+Bl9ObDg==" + "integrity": "sha512-n6wvpdE43VFtJq+lUDYDBFUwV8TZbuGXLV4D6wKafg13ldznKsyEvatubnmUe31zcvelSzOHF+XbaT+Bl9ObDg==", + "optional": true } } }, @@ -2754,7 +2768,8 @@ "esprima": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/esprima/-/esprima-3.1.3.tgz", - "integrity": "sha1-/cpRzuYTOJXjyI1TXOSdv/YqRjM=" + "integrity": "sha1-/cpRzuYTOJXjyI1TXOSdv/YqRjM=", + "optional": true }, "esquery": { "version": "1.0.1", @@ -3429,7 +3444,8 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "aproba": { "version": "1.2.0", @@ -3450,12 +3466,14 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -3470,17 +3488,20 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "core-util-is": { "version": "1.0.2", @@ -3597,7 +3618,8 @@ "inherits": { "version": "2.0.3", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "ini": { "version": "1.3.5", @@ -3609,6 +3631,7 @@ "version": "1.0.0", "bundled": true, "dev": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -3623,6 +3646,7 @@ "version": "3.0.4", "bundled": true, "dev": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -3630,12 +3654,14 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "minipass": { "version": "2.2.4", "bundled": true, "dev": true, + "optional": true, "requires": { "safe-buffer": "^5.1.1", "yallist": "^3.0.0" @@ -3654,6 +3680,7 @@ "version": "0.5.1", "bundled": true, "dev": true, + "optional": true, "requires": { "minimist": "0.0.8" } @@ -3741,7 +3768,8 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "object-assign": { "version": "4.1.1", @@ -3753,6 +3781,7 @@ "version": "1.4.0", "bundled": true, "dev": true, + "optional": true, "requires": { "wrappy": "1" } @@ -3838,7 +3867,8 @@ "safe-buffer": { "version": "5.1.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "safer-buffer": { "version": "2.1.2", @@ -3874,6 +3904,7 @@ "version": "1.0.2", "bundled": true, "dev": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -3893,6 +3924,7 @@ "version": "3.0.1", "bundled": true, "dev": true, + "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -3936,12 +3968,14 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "yallist": { "version": "3.0.2", "bundled": true, - "dev": true + "dev": true, + "optional": true } } }, @@ -5341,7 +5375,8 @@ "hoek": { "version": "2.16.3", "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz", - "integrity": "sha1-ILt0A9POo5jpHcRxCo/xuCdKJe0=" + "integrity": "sha1-ILt0A9POo5jpHcRxCo/xuCdKJe0=", + "optional": true }, "homedir-polyfill": { "version": "1.0.1", @@ -5438,6 +5473,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-2.1.0.tgz", "integrity": "sha512-qwHbBLV7WviBl0rQsOzH6o5lwyOIvwp/BdFnvVxXORldu5TmjFfjzBcWUWS5kWAZhmv+JtiDhSuQCp4sBfbIgg==", + "optional": true, "requires": { "agent-base": "4", "debug": "3.1.0" @@ -5447,6 +5483,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", + "optional": true, "requires": { "ms": "2.0.0" } @@ -5467,6 +5504,7 @@ "version": "1.6.1", "resolved": "https://registry.npmjs.org/httpntlm/-/httpntlm-1.6.1.tgz", "integrity": "sha1-rQFScUOi6Hc8+uapb1hla7UqNLI=", + "optional": true, "requires": { "httpreq": ">=0.4.22", "underscore": "~1.7.0" @@ -5475,14 +5513,16 @@ "underscore": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz", - "integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk=" + "integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk=", + "optional": true } } }, "httpreq": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/httpreq/-/httpreq-0.4.24.tgz", - "integrity": "sha1-QzX/2CzZaWaKOUZckprGHWOTYn8=" + "integrity": "sha1-QzX/2CzZaWaKOUZckprGHWOTYn8=", + "optional": true }, "https-browserify": { "version": "1.0.0", @@ -5493,6 +5533,7 @@ "version": "2.2.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz", "integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==", + "optional": true, "requires": { "agent-base": "^4.1.0", "debug": "^3.1.0" @@ -5502,6 +5543,7 @@ "version": "3.2.6", "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "optional": true, "requires": { "ms": "^2.1.1" } @@ -5509,7 +5551,8 @@ "ms": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", + "optional": true } } }, @@ -5717,7 +5760,8 @@ "ip": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", - "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=" + "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=", + "optional": true }, "ipaddr.js": { "version": "1.8.0", @@ -5982,7 +6026,8 @@ "is-property": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", - "integrity": "sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ=" + "integrity": "sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ=", + "optional": true }, "is-regex": { "version": "1.0.4", @@ -6257,12 +6302,14 @@ "libbase64": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/libbase64/-/libbase64-0.1.0.tgz", - "integrity": "sha1-YjUag5VjrF/1vSbxL2Dpgwu3UeY=" + "integrity": "sha1-YjUag5VjrF/1vSbxL2Dpgwu3UeY=", + "optional": true }, "libmime": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/libmime/-/libmime-3.0.0.tgz", "integrity": "sha1-UaGp50SOy9Ms2lRCFnW7IbwJPaY=", + "optional": true, "requires": { "iconv-lite": "0.4.15", "libbase64": "0.1.0", @@ -6272,14 +6319,16 @@ "iconv-lite": { "version": "0.4.15", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.15.tgz", - "integrity": "sha1-/iZaIYrGpXz+hUkn6dBMGYJe3es=" + "integrity": "sha1-/iZaIYrGpXz+hUkn6dBMGYJe3es=", + "optional": true } } }, "libqp": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/libqp/-/libqp-1.1.0.tgz", - "integrity": "sha1-9ebgatdLeU+1tbZpiL9yjvHe2+g=" + "integrity": "sha1-9ebgatdLeU+1tbZpiL9yjvHe2+g=", + "optional": true }, "liftoff": { "version": "2.5.0", @@ -7499,12 +7548,14 @@ "nodemailer-fetch": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/nodemailer-fetch/-/nodemailer-fetch-1.6.0.tgz", - "integrity": "sha1-ecSQihwPXzdbc/6IjamCj23JY6Q=" + "integrity": "sha1-ecSQihwPXzdbc/6IjamCj23JY6Q=", + "optional": true }, "nodemailer-shared": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/nodemailer-shared/-/nodemailer-shared-1.1.0.tgz", "integrity": "sha1-z1mU4v0mjQD1zw+nZ6CBae2wfsA=", + "optional": true, "requires": { "nodemailer-fetch": "1.6.0" } @@ -7534,7 +7585,8 @@ "nodemailer-wellknown": { "version": "0.1.10", "resolved": "https://registry.npmjs.org/nodemailer-wellknown/-/nodemailer-wellknown-0.1.10.tgz", - "integrity": "sha1-WG24EB2zDLRDjrVGc3pBqtDPE9U=" + "integrity": "sha1-WG24EB2zDLRDjrVGc3pBqtDPE9U=", + "optional": true }, "nopt": { "version": "3.0.6", @@ -9457,6 +9509,11 @@ "resolve": "^1.1.6" } }, + "redaxios": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/redaxios/-/redaxios-0.2.0.tgz", + "integrity": "sha512-lrDld2bVWIBrW+S1HPMyP8OvEtpHPIfmXyW7T7xTox1SM4/M3aC5X55xa2vRR3LZHwCvpMQAgCye8IyICzkKpA==" + }, "redent": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/redent/-/redent-1.0.0.tgz", @@ -10260,12 +10317,14 @@ "smart-buffer": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.0.2.tgz", - "integrity": "sha512-JDhEpTKzXusOqXZ0BUIdH+CjFdO/CR3tLlf5CN34IypI+xMmXW1uB16OOY8z3cICbJlDAVJzNbwBhNO0wt9OAw==" + "integrity": "sha512-JDhEpTKzXusOqXZ0BUIdH+CjFdO/CR3tLlf5CN34IypI+xMmXW1uB16OOY8z3cICbJlDAVJzNbwBhNO0wt9OAw==", + "optional": true }, "smtp-connection": { "version": "2.12.0", "resolved": "https://registry.npmjs.org/smtp-connection/-/smtp-connection-2.12.0.tgz", "integrity": "sha1-1275EnyyPCJZ7bHoNJwujV4tdME=", + "optional": true, "requires": { "httpntlm": "1.6.1", "nodemailer-shared": "1.1.0" @@ -10391,6 +10450,7 @@ "version": "2.2.3", "resolved": "https://registry.npmjs.org/socks/-/socks-2.2.3.tgz", "integrity": "sha512-+2r83WaRT3PXYoO/1z+RDEBE7Z2f9YcdQnJ0K/ncXXbV5gJ6wYfNAebYFYiiUjM6E4JyXnPY8cimwyvFYHVUUA==", + "optional": true, "requires": { "ip": "^1.1.5", "smart-buffer": "4.0.2" @@ -10400,6 +10460,7 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-4.0.1.tgz", "integrity": "sha512-Kezx6/VBguXOsEe5oU3lXYyKMi4+gva72TwJ7pQY5JfqUx2nMk7NXA6z/mpNqIlfQjWYVfeuNvQjexiTaTn6Nw==", + "optional": true, "requires": { "agent-base": "~4.2.0", "socks": "~2.2.0" diff --git a/package.json b/package.json index 60b0f41..7d71e47 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "test": "mocha" }, "engines": { - "node": ">=8.9.1" + "node": "=8.17.0" }, "author": "", "license": "ISC", @@ -16,6 +16,7 @@ "backbone": "^1.3.3", "browserify": "^16.2.3", "cheerio": "^1.0.0-rc.2", + "cors": "^2.8.5", "dark-sky-api": "^0.6.32", "dateformat": "^3.0.3", "debug-logger": "^0.4.1", @@ -49,6 +50,7 @@ "muicss": "^0.9.41", "node-foursquare-venues": "^1.1.0", "openweather-apis": "^3.3.5", + "redaxios": "^0.2.0", "request-json": "^0.6.4", "request-promise-native": "^1.0.5", "strict-uri-encode": "^2.0.0", diff --git a/server.js b/server.js index 1352a51..456fac1 100644 --- a/server.js +++ b/server.js @@ -10,9 +10,14 @@ const agenda = require('./server/agenda'); const directions = require('./server/directions'); const geocode = require('./server/geocode'); +const cors = require('cors'); + logger.level = 'debug'; const app = express(); + +app.use(cors()); + const port = process.env.PORT || 8110; const sitePath = 'live'; diff --git a/server/reducers/euronews.js b/server/reducers/euronews.js index 743f803..49890fc 100644 --- a/server/reducers/euronews.js +++ b/server/reducers/euronews.js @@ -20,7 +20,13 @@ function reduceArticle(body = '') { const html = []; - const content = $('div.article__content'); + const content = $('div.c-article-content'); + const ampAddStripper = /<\s*amp-ad(\s+.*?>|>).*?<\s*\/\s*amp-ad\s*>/ig; + + $(content).find('amp-ad').remove(); + + $(content).find('div.widget__wrapper').remove(); + $(content).find('div.widget').remove(); for (let top = 0, topLen = content.length; top < topLen; top++) { const children = $(content[top]).children(); @@ -31,6 +37,8 @@ function reduceArticle(body = '') { const symbol = /src=(['"])(http[s]?:\/\/)/.exec(line) || []; + + if (tag === 'amp-twitter') { const tweetid = $(children[index]).data('tweetid'); line = `|>).*?<\s*\/\s*amp-ad\s*>/ig; + + $(content).find('amp-ad').remove(); + + $(content).find('div.widget__wrapper').remove(); for (let top = 0, topLen = content.length; top < topLen; top++) { const children = $(content[top]).children(); diff --git a/src/v1/js/Weather.js b/src/v1/js/Weather.js index 20b291e..1d32265 100644 --- a/src/v1/js/Weather.js +++ b/src/v1/js/Weather.js @@ -124,10 +124,10 @@ const WeatherModel = Backbone.Model.extend({ } }.bind(this)); }, - 'logUpdate': function() { + 'logUpdate': function () { console.log('Weather logging:'); - const log = { 'lat' : this.get('latitude'), 'long': this.get('longitude'), 'time': new Date().getTime() }; + const log = { 'lat': this.get('latitude'), 'long': this.get('longitude'), 'time': new Date().getTime() }; this.set('log', log, { 'silent': true }); diff --git a/test/data/euronews/20200506-widgets.html b/test/data/euronews/20200506-widgets.html new file mode 100644 index 0000000..c9315cf --- /dev/null +++ b/test/data/euronews/20200506-widgets.html @@ -0,0 +1,965 @@ + + + + + + + VE Day: A locked down Europe commemorates as best it can | Euronews + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+
+
+

+ VE Day: A locked down Europe commemorates as best it can +

+
+ By Euronews  with AP + +
+ + + +
+
+ +
+

Countries across Europe are marking the 75th anniversary of the Nazi regime's surrender as best they can while most of the Old Continent remains under lockdown due to the COVID-19 pandemic.

+ +
+
+
+
+
+
+
+
+ + +

In the UK, commemorations started with a military flypast over London shortly after 11:00 CEST.

+ +
+
+
+
+ +

A nationwide two-minute silence to honour those who made the ultimate sacrifice was then held at 12:00 CEST, led by Prince Charles and the Duchess of Cornwall.

+
+
+
+
+ +

The Queen will deliver a televised speech at 22:00 CEST — the exact time her father addressed the nation 75 years ago.

+

In France, President Emmanuel Macron laid a wreath in front of the statue of General Charles de Gaulle in Paris before attending a ceremony at the Arc de Triomphe. The ceremony, also attended by former leaders Francois Hollande and Nicolas Sarkozy, has been closed to the public because of the pandemic.

+ +
+
+
French President Emmanuel Macron, center right, lays a wreath of flowers at the statue of General Charles de Gaulle during VE Day ceremonies Friday May 8, 2020 in Paris. + AP Photo/Francois Mori, Pool + Francois Mori +
+
+ + +

'A day of liberation' for Germany

German Chancellor Angela Merkel will also lay a wreath to mark the occasion at the Central Memorial of the Federal Republic of Germany for the Victims of War and Tyranny.

+

Berlin's Brandenburg Gate is also expected to be illuminated. The German capital has turned the commemoration into a public holiday sparking a debate in the country as to whether Victory Day should become an annual federal public holiday.

+

German President Frank-Walter Steinmeier said in a speech that on May 8, 1945, the country was alone.

+

"Germany had suffered military defeat, political and economic ruin, and moral collapse. We had made enemies of the entire world," he said.

+

He added that the country is now "forced to commemorate alone" because of the pandemic, but we are not alone!"

+

"We live in vigorous and well-established democracy, in a country that has been reunified for 30 years, at the heart of a peaceful and united Europe. We are a trusted member of the international community and reap the fruits of cooperation and partnership around the world.

+

"It has taken three generations for us to admit wholeheartedly: 8 May 1945 was indeed a day of liberation.

+

"Seventy-five years after the end of the war, we Germans have much to be thankful for. But none of the positive achievements since that data are safe in perpetuity. May 8 was not the end of the liberation — preserving freedom and democracy is the never-ending task that has been bequeathed upon us," he stressed.

+

Russian military parade postponed

In Russia, the traditional Victory Day military parade, which was to be held on May 9 has been postponed to later this year due to the pandemic. The event was to be attended by Macron and Chinese President Xi Jinping.

+

Zinaida Korneva, a 97-year-old veteran hoping to replicate the fundraising prowess achieved in the UK by Captain Tom Moore, has taken to YouTube to talk about her trials as a Red Army soldier in the Stalingrad region.

+

Money she collects will be donated to the families of doctors fighting COVID-19 in St Petersburg.

+
+
+
+
+
+ +
+

+ You might also like +

+ +
+ + + +
+ + + + +
+
+ +
+

+ Top videos +

+ + + + +
+
+ +
+ +
+ + + + + + + + + + + + + + + +
+ + +
+ + + diff --git a/test/data/euronews/20200506ampRedirectedToHtml.html b/test/data/euronews/20200506ampRedirectedToHtml.html new file mode 100644 index 0000000..46d51e2 --- /dev/null +++ b/test/data/euronews/20200506ampRedirectedToHtml.html @@ -0,0 +1,2263 @@ + + + + + + + + + + + + + + + + + + + + Coronavirus latest: EU expects recession of 'historic proportions' as UK deaths highest in Europe | Euronews + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+
+
+
+
+
+
+ +
+ +
+ +
+
+
+
+
+
+ +
+ +
+
+ +
+
+
+
BREAKING NEWS +
+
+
+
+ +
+
+
+
+
+
+ +
+
+
+
+
+ +
+
+ +
+
+
+ This content is not available in your region +
+
+
+
+
+
+

World

+ +

Coronavirus latest: EU expects recession of 'historic + proportions' as UK deaths highest in Europe

Comments +
+
+
+ + By Orlando Crowcroft + with AP + +
+
+
+
+
+
+
+ + euronews_icons_loading + + + +
+
+
+
+ +
+
+
+
+
+ A man wears a makeshift face mask during the coronavirus lockdown in London, Tuesday, May 5, 2020. +
+
+
+ A man wears a makeshift face mask during the coronavirus lockdown in London, Tuesday, May 5, 2020. + +   -   + Copyright  + Frank Augstein/Copyright 2020 The Associated Press. All rights reserved
+
+
+ +
+
+ Text size + Aa + + Aa +
+
+
+

Summary of key developments

+

**

+
+
+
+
+
+ +

**

+
+
+ +
+

Latest updates

+

+

+
+ +
+
+ + +

+
+
+ +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+ +
+ + +
+
+
+
+
+ + + + + + + + + + + + + + + + diff --git a/test/data/euronews/20200506chinaspace.html b/test/data/euronews/20200506chinaspace.html new file mode 100644 index 0000000..526767e --- /dev/null +++ b/test/data/euronews/20200506chinaspace.html @@ -0,0 +1,925 @@ + + + + + + + China says its new space station will be completed by 2022 | Euronews + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+
+
+

+ China says its new space station will be completed by 2022 +

+
+ By Euronews  with AP + +
+ + + +
+
+ +
+

China is aiming to complete work on its space station within about two years, officials said after the launch of a new spacecraft on Tuesday.

+ +

The country launched a new rocket from the Wenchang launch centre with an unmanned spacecraft and return capsule.

+ +

The launch was successful, China's Manned Space Agency said, as the experimental spacecraft and capsule separated from the rocket to enter orbit.

+

The capsule is reportedly based on the former Soviet Union's Soyuz model and can carry six astronauts rather than three.

+

China eventually plans to send four crewed space missions and cargo craft to complete work on its permanent space station.

+ +

Rapid development of China's program

China's space program has developed rapidly since its first crewed mission in 2003, including achieving a milestone last year by landing a spacecraft on the dark side of the moon.

+

Although China has sought cooperation with space agencies in Europe and elsewhere, it is banned from space cooperation due to US national security concerns that has kept the country out of the International Space Station.

+

China's experimental space station crashed into the Pacific Ocean in 2018 but it now plans to build a larger facility.

+

China is among three countries planning missions to Mars for this summer along with the United States and United Arab Emirates.

+

Spacecraft can only be launched to Mars every two years, to take advantage of the best possible lineup between Earth and its neighbouring planet.

+
+ +
+

+ You might also like +

+ +
+ + + +
+ + + + +
+
+ +
+

+ Top videos +

+ + + + +
+
+ +
+ +
+ + + + + + + + + + + + + + + +
+ + +
+ + + diff --git a/test/data/euronews/20200506impact.html b/test/data/euronews/20200506impact.html new file mode 100644 index 0000000..1b34bd9 --- /dev/null +++ b/test/data/euronews/20200506impact.html @@ -0,0 +1,957 @@ + + + + + + + The negative impact of the pandemic on the mental health of NHS staff is being ignored ǀ View | Euronews + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+
+
+

+ The negative impact of the pandemic on the mental health of NHS staff is being ignored ǀ View +

+
+ By Hadley Stewart +
+ + + +
+
+
+
+ Opinions expressed in View articles are solely those of the authors. + + +
+
+ +
+

The United Kingdom’s National Health Service (NHS) is in the midst of arguably the most challenging chapter of its history. The health service, alongside international counterparts, is grappling with the effects of the coronavirus pandemic on the public’s health. Over 26,000 people have now lost their lives to the virus in the UK, including more than 100 NHS staff. Whilst the focus has been on the provision of personal protective equipment (PPE) and swiftly expanding bed capacity in the NHS to meet potential future needs, the mental health and emotional well-being of clinicians has been somewhat of an afterthought. I would argue that the vulnerabilities of NHS staff go beyond the physical health threat from the virus.

+ +

According to the most-recent NHS staff survey published last year, 40% of respondents said that they felt stressed at work. Although the survey didn’t go further into exploring other mental health issues, the Health Service Journal reported that an estimated 348,028 working days had been lost due to stress, anxiety and depression. The working environment in the NHS today looks very different to how it did before the coronavirus outbreak was declared a pandemic. There has been a sharp increase in demand on the health service, healthcare professionals are having to work in PPE for long-periods of time, and the number of patients dying is also on the increase. These factors all contribute to poor mental health amongst clinicians.

+ +

After a decade of government-led austerity, the focus of healthcare funding has been to play catch-up on the provision of resources; in the form of PPE, ventilators and critical care bed capacity. Though these are all vital tools to weather this storm, I fear that not enough was done to prepare NHS staff for the emotional toll this crisis would have on them. I think it was an important step for the NHS to introduce mental health support at the beginning of April, yet these interventions might already be too late for a system of professionals who are already seeing high rates of poor mental health and workplace stress. The coronavirus pandemic will only widen the cracks in the mental health of NHS staff.

+
+
+
+ There needs to be a broader conversation about mental health among healthcare professionals during this pandemic, which needs to be instigated by leaders of government and NHS employers. +
+
Hadley Stewart
+
Writer and medical journalist
+
+
+ + +

The notion of “NHS Heroes” is a double-sided coin when it comes to the mental health of clinical staff. On the one hand, the fact that the public admiration of the NHS and those that work in the organisation has only grown since the onset of the pandemic is extremely positive for morale. However, this notion of heroism also puts clinicians on a pedestal. I would argue that the framing of doctors and nurses as such only adds to the pressure to carry on working, despite feeling physically and emotionally fatigued. As a consequence, clinical staff might feel unable to ask for support from their colleagues or their support network outside of work. This puts undue pressure on them to sometimes continue working at the detriment of their mental health. I think it’s important that this concept of NHS staff being heroic doesn’t deter their ability to look after their own mental health and emotional wellbeing.

+

Social distancing measures are also taking their toll. Things like visiting family and friends, looking forward to holidays, or just going out for a meal at their favourite restaurant now feel like distant memories. Working in the NHS has never been easy, especially in recent years with an ageing population with increasingly complex health needs. However, the coronavirus pandemic has not only added to the challenges of working in the health sector, it has also closed off many avenues of support and emotional release. Not having plans to look forward to whilst doing a job which is both physically and emotionally challenging will doubtless have an long-term impacts on rates of burnout, staff turnover and vacancy rates.

+ +
+
+
+ If the discourse surrounding working conditions and well-being isn’t opened up to include the mental health impacts of the coronavirus pandemic, the NHS workforce will suffer in the long-run. +
+
Hadley Stewart
+
Writer and medical journalist
+
+
+ + +

I don’t think there are any easy ways to overcome the hurdles that healthcare professionals are facing in the NHS and in health systems across the world. Setting up specific mental health support services for NHS staff is a start but this is hardly a new concept. Activities in the workplace that promote positive mental health could also be an opportunity for frontline staff to take some time away from their stressful working environments. Equally, empowering staff to look after their emotional well-being away from the workplace by educating them about techniques like mindfulness or meditation could help combat the effects of social distancing. But I think ultimately, there needs to be a broader conversation about mental health among healthcare professionals during this pandemic, which needs to be instigated by leaders of government and NHS employers.

+

The focus of the UK government during the coronavirus pandemic has been on the provision of physical resources, such as PPE and ventilators. These are relatively easy to both implement and to explain to the public, because there is a numerical value attached to these interventions. The promotion and improvement of emotional well-being of NHS staff, however, is a little more abstract and not as straightforward. That being said, this shouldn’t make the mental health of NHS staff any less important than their physical well-being. Yes, this virus has tragically killed many frontline staff but the mental health cost of this pandemic is yet to be known. What is clear, though, is that if the discourse surrounding working conditions and well-being isn’t opened up to include the mental health impacts of the coronavirus pandemic, the NHS workforce will suffer in the long-run.

+
  • Hadley Stewart is a London-based writer, broadcaster and medical journalist.
    • +

____________

+

Are you a recognised expert in your field? At Euronews, we believe all views matter. Contact us at view@euronews.com to send pitches or submissions and be part of the conversation.

+
+ +
+

+ You might also like +

+ +
+ + + +
+ + + + +
+
+ +
+

+ Top videos +

+ + + + +
+
+ +
+ +
+ + + + + + + + + + + + + + + +
+ + +
+ + + diff --git a/test/data/euronews/20200506lifelockdown.html b/test/data/euronews/20200506lifelockdown.html new file mode 100644 index 0000000..688b494 --- /dev/null +++ b/test/data/euronews/20200506lifelockdown.html @@ -0,0 +1,934 @@ + + + + + + + Analysis: Will life after lockdown really be so different? | Euronews + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + +
+
+
+

+ Analysis: Will life after lockdown really be so different? +

+
+ By Darren McCaffrey, Political Editor +
+ + + +
+
+ +
+

The economic data are not just bad, but terrible. GDP slumping by record levels, millions relying on government schemes, businesses going bust, Europe facing its worst recession for the best part of a century.

+ +

But there is hope that — with a vaccine, possible drugs and accepted social distancing, coupled with massive state subsidies — economies can recover relatively quickly and this might not turn into a 1930s depression.

+ +

But even if this is true — and it really is a big if — the current lockdowns and the economic hangover may well change our habits forever.

+

Now, don’t get me wrong. Lots of people have written lots of things about how the coronavirus crisis will change the world. Frankly, it does make a good read. But I’m not sure the world will fundamentally shift; evidence coming out of China so far suggests most things are returning to normal.

+

The crisis of 2020 may exacerbate current trends, though — not least of all the growing influence and power of an authoritarian China. But it’s unlikely we will be suddenly seeing socialist revolutions. However, there are things that will change.

+ +

Firstly, in a recession they always do. Most people will have less money, or will at least be more concerned about what they spend their money on.

+

Discretionary spending in China is down 40 per cent from a year ago.

+

How much were you spending on takeaway coffee a couple of months ago? Will you keep paying €5 for your vanilla coconut milk cappuccino with no foam and whipped cream every day? Do we need to replace our cars quite so often?

+

Today the UK reported the lowest car sales in April in 74 years. And with restaurants, pubs and theatres likely to remain closed or partially closed for many months to come, we can’t help but spend less money. This is pretty obvious.

+

So what about when things return to what looks like the “old normal” and growth returns? Well, we have to remember that this isn’t like previous recessions. This one was brought on by a virus; a virus that forced us to stay indoors. It created fear.

+

One of the winners in recent weeks have been takeaway companies, delivering not just cheap pizza and burgers but actual three-course meals from some of our favourite restaurants. Why plod into town in the future, when you can enjoy the same tasty steak without the usual fuss at home? The same goes for our weekly shop. Millions of people are now having groceries delivered for the first time or buying other essential items online. It’s a new habit many will keep.

+

Cash was said to be king. In Germany and other parts of Europe, that was frustratingly true. But, shops are increasingly insisting on payment by card only. Personally, I haven’t used cash for weeks. I bet you most of those shops will now keep those “card only” signs in the window. These are only a few small examples, added to some bigger, perhaps more profound ones.

+

Will many more of us work from home? Germany and Finland have now enshrined in law that this should be an option for employees. Surely, lockdown arrangements have proven we don’t all need a desk in big, high rise office blocks? We can do our jobs effectively and efficiently from home. And what knock-on impact on our big metropolitan cities?

+

Also, it seems lots more people will take up cycling, walking or scooting to work, to avoid public transport. Cities like Brussels are now insisting on lower speed limits and more bicycle lanes being imposed this summer. And if travelling abroad becomes expensive and painfully slow due to extra checks, might we decide on longer, less frequent holidays, closer to home? Are the European weekend city breaks a thing of the past?

+

We obviously don’t know for sure, and while the world is unlikely to be turned on its head, this unprecedented crisis will have some impact. The new, “new normal” will not like the old normal.

+

Darren McCaffrey is Euronews' political editor.

+
+ +
+

+ You might also like +

+ +
+ + + +
+ + + + +
+
+ +
+

+ Top videos +

+ + + + +
+
+ +
+ +
+ + + + + + + + + + + + + + + +
+ + +
+ + + diff --git a/test/euronews.scrape.js b/test/euronews.scrape.js index 239ab2d..7e4c7fc 100644 --- a/test/euronews.scrape.js +++ b/test/euronews.scrape.js @@ -7,14 +7,14 @@ const jsonfile = require('jsonfile'); const cheerio = require('cheerio'); -const { reduceArticle,reduceArticleV2 } = require('../server/reducers/euronews'); +const { reduceArticle, reduceArticleV2 } = require('../server/reducers/euronews'); + +const psDetail = fs.readFileSync('test/data/euronews/20200506-widgets.html'); test('Euronews', async t => { - t.skip('Reduce a page', async t => { - const psDetail = fs.readFileSync('./data/euronews/amp-20200114.html'); + t.test('Reduce a page', async t => { const expectedJSON = { 'bob':false }; // jsonfile.readFileSync('tests/data/cz/ps001.json'); - const output = await reduceArticle(psDetail); t.deepEquals(output, expectedJSON, 'Extracted Details from Page'); @@ -22,19 +22,15 @@ test('Euronews', async t => { t.end(); }); - t.test('Reduce a page V2', async t => { - const psDetail = fs.readFileSync('./data/euronews/amp-20200114.html'); - const expectedJSON = { 'bob':false }; // jsonfile.readFileSync('tests/data/cz/ps001.json'); + const expectedJSON = { 'bob':false }; // jsonfile.readFileSync('tests/data/cz/ps001.json'); + const output = await reduceArticleV2(psDetail); - const output = await reduceArticleV2(psDetail); - - t.deepEquals(output, expectedJSON, 'Extracted Details from Page'); - - t.end(); - }); + t.deepEquals(output, expectedJSON, 'Extracted Details from Page'); + t.end(); + }); t.end(); });