From ab56fae4cdcd6727900eeeb362134771299bafbf Mon Sep 17 00:00:00 2001 From: Martin Donnelly Date: Mon, 8 Apr 2019 14:44:12 +0100 Subject: [PATCH] updated news feeds --- .env | 4 + .idea/jsLibraryMappings.xml | 1 - .idea/workspace.xml | 276 +++++++++++++++++++++--------------- feeds/news.json | 24 ---- gather.js | 6 + package-lock.json | 50 +++++-- package.json | 1 + 7 files changed, 208 insertions(+), 154 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 0000000..16ea613 --- /dev/null +++ b/.env @@ -0,0 +1,4 @@ + +PROXY_URI=uk.proxymesh.com:31280 + +LOGGER_LEVEL=info diff --git a/.idea/jsLibraryMappings.xml b/.idea/jsLibraryMappings.xml index 683f4b8..d23208f 100644 --- a/.idea/jsLibraryMappings.xml +++ b/.idea/jsLibraryMappings.xml @@ -1,7 +1,6 @@ - \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 9cde519..5eb3ce5 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,8 +2,13 @@ + + + + + @@ -24,34 +29,38 @@ - + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + @@ -60,16 +69,16 @@ - + - + - - + + @@ -122,12 +131,6 @@ - processJson - worki - techno - /!* "name" : "JobServe",*!/ - address - /* "name" : "JobServe",*/ news jobis jobisjob @@ -152,6 +155,12 @@ jobs mensf http%3A%2F%2Fwww.purelyit.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D3325606 + request + proxy + http%3A%2F%2Fwww.deadlinenews.co.uk%2Ffeed + http://45.33.114.116:8080/http%3A%2F%2Fwww.deadlinenews.co.uk%2Ffeed + edia + london @@ -165,6 +174,7 @@ C:\dev\md\Rinser $PROJECT_DIR$/html/js + $PROJECT_DIR$/feeds @@ -174,7 +184,6 @@ @@ -236,10 +246,10 @@ true - @@ -254,6 +264,11 @@ + + + + + @@ -284,11 +299,13 @@ + - + + @@ -306,12 +323,16 @@ - + - + + + + + @@ -319,10 +340,6 @@ - - - - - - + - + - + @@ -879,7 +903,7 @@ - + @@ -887,6 +911,7 @@ + @@ -1030,15 +1055,6 @@ - - - - - - - - - @@ -1118,20 +1134,6 @@ - - - - - - - - - - - - - - @@ -1271,6 +1273,68 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1299,41 +1363,17 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + diff --git a/feeds/news.json b/feeds/news.json index 1b9c11a..daef509 100644 --- a/feeds/news.json +++ b/feeds/news.json @@ -57,10 +57,6 @@ "count": 100, "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.cityam.com%2Ffeeds%2Fmain.xml" }, - { - "count": 100, - "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.londonlovesbusiness.com%2Fnavrss%3Fnavsectioncode%3D121" - }, { "count": 100, "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.ibtimes.co.uk%2Frss%2Fuk" @@ -120,25 +116,5 @@ { "count": 100, "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Ftheconversation.com%2Fuk%2Farticles.atom" - }, - { - "count": 100, - "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.deadlinenews.co.uk%2Ffeed" - }, - { - "count": 100, - "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.deadlinenews.co.uk%2Ffeed" - }, - { - "count": 100, - "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.deadlinenews.co.uk%2Ffeed" - }, - { - "count": 100, - "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fnewsnet.scot%2Ffeed%2F" - }, - { - "count": 100, - "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fswd.media%2Ffeed%2F" } ] } diff --git a/gather.js b/gather.js index 86fdc36..a1b069c 100644 --- a/gather.js +++ b/gather.js @@ -25,6 +25,12 @@ feeds.tech = require('./feeds/tech.json'); feeds.fit = require('./feeds/fit.json'); feeds.bored = require('./feeds/bored.json'); +require('dotenv').config({ + 'path': `${__dirname }/../.env` +}); + + + const braiderOptions = { feeds, 'indent': ' ', diff --git a/package-lock.json b/package-lock.json index ebcb928..cdd4a47 100644 --- a/package-lock.json +++ b/package-lock.json @@ -191,6 +191,7 @@ "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.2.0.tgz", "integrity": "sha512-c+R/U5X+2zz2+UCrCFv6odQzJdoqI+YecuhnAJLa1zYaMc13zPfwMwZrr91Pd1DYNo/yPRbiM4WVf9whgwFsIg==", "dev": true, + "optional": true, "requires": { "es6-promisify": "^5.0.0" } @@ -704,7 +705,8 @@ "version": "0.0.2", "resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-0.0.2.tgz", "integrity": "sha1-JrOIXRD6E9t/wBquOquHAZngEkw=", - "dev": true + "dev": true, + "optional": true }, "bufferstreams": { "version": "1.1.3", @@ -1638,6 +1640,11 @@ "domelementtype": "1" } }, + "dotenv": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-6.2.0.tgz", + "integrity": "sha512-HygQCKUBSFl8wKQZBSemMywRWcEDNidvNbjGVyZu3nbZ8qq9ubiPoGLMdRDpfSrpkkm9BXYFkpKxxFX38o/76w==" + }, "double-ended-queue": { "version": "2.1.0-0", "resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz", @@ -1738,13 +1745,15 @@ "version": "4.2.4", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.4.tgz", "integrity": "sha512-/NdNZVJg+uZgtm9eS3O6lrOLYmQag2DjdEXuPaHlZ6RuVqgqaVZfgYCepEIKsLqwdQArOPtC3XzRLqGGfT8KQQ==", - "dev": true + "dev": true, + "optional": true }, "es6-promisify": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", "dev": true, + "optional": true, "requires": { "es6-promise": "^4.0.3" } @@ -4016,6 +4025,7 @@ "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-2.1.0.tgz", "integrity": "sha512-qwHbBLV7WviBl0rQsOzH6o5lwyOIvwp/BdFnvVxXORldu5TmjFfjzBcWUWS5kWAZhmv+JtiDhSuQCp4sBfbIgg==", "dev": true, + "optional": true, "requires": { "agent-base": "4", "debug": "3.1.0" @@ -4026,6 +4036,7 @@ "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", "dev": true, + "optional": true, "requires": { "ms": "2.0.0" } @@ -4047,6 +4058,7 @@ "resolved": "https://registry.npmjs.org/httpntlm/-/httpntlm-1.6.1.tgz", "integrity": "sha1-rQFScUOi6Hc8+uapb1hla7UqNLI=", "dev": true, + "optional": true, "requires": { "httpreq": ">=0.4.22", "underscore": "~1.7.0" @@ -4056,13 +4068,15 @@ "version": "0.4.24", "resolved": "https://registry.npmjs.org/httpreq/-/httpreq-0.4.24.tgz", "integrity": "sha1-QzX/2CzZaWaKOUZckprGHWOTYn8=", - "dev": true + "dev": true, + "optional": true }, "https-proxy-agent": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz", "integrity": "sha512-HPCTS1LW51bcyMYbxUIOO4HEOlQ1/1qRaFWcyxvwaqUS9TY88aoEuHUY33kuAh1YhVVaDQhLZsnPd+XNARWZlQ==", "dev": true, + "optional": true, "requires": { "agent-base": "^4.1.0", "debug": "^3.1.0" @@ -4073,6 +4087,7 @@ "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", "dev": true, + "optional": true, "requires": { "ms": "2.0.0" } @@ -4230,7 +4245,8 @@ "version": "1.1.5", "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=", - "dev": true + "dev": true, + "optional": true }, "ipaddr.js": { "version": "1.6.0", @@ -4652,13 +4668,15 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/libbase64/-/libbase64-0.1.0.tgz", "integrity": "sha1-YjUag5VjrF/1vSbxL2Dpgwu3UeY=", - "dev": true + "dev": true, + "optional": true }, "libmime": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/libmime/-/libmime-3.0.0.tgz", "integrity": "sha1-UaGp50SOy9Ms2lRCFnW7IbwJPaY=", "dev": true, + "optional": true, "requires": { "iconv-lite": "0.4.15", "libbase64": "0.1.0", @@ -4669,7 +4687,8 @@ "version": "0.4.15", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.15.tgz", "integrity": "sha1-/iZaIYrGpXz+hUkn6dBMGYJe3es=", - "dev": true + "dev": true, + "optional": true } } }, @@ -4677,7 +4696,8 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/libqp/-/libqp-1.1.0.tgz", "integrity": "sha1-9ebgatdLeU+1tbZpiL9yjvHe2+g=", - "dev": true + "dev": true, + "optional": true }, "liftoff": { "version": "2.3.0", @@ -5707,13 +5727,15 @@ "version": "1.6.0", "resolved": "https://registry.npmjs.org/nodemailer-fetch/-/nodemailer-fetch-1.6.0.tgz", "integrity": "sha1-ecSQihwPXzdbc/6IjamCj23JY6Q=", - "dev": true + "dev": true, + "optional": true }, "nodemailer-shared": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/nodemailer-shared/-/nodemailer-shared-1.1.0.tgz", "integrity": "sha1-z1mU4v0mjQD1zw+nZ6CBae2wfsA=", "dev": true, + "optional": true, "requires": { "nodemailer-fetch": "1.6.0" } @@ -5746,7 +5768,8 @@ "version": "0.1.10", "resolved": "https://registry.npmjs.org/nodemailer-wellknown/-/nodemailer-wellknown-0.1.10.tgz", "integrity": "sha1-WG24EB2zDLRDjrVGc3pBqtDPE9U=", - "dev": true + "dev": true, + "optional": true }, "normalize-path": { "version": "2.1.1", @@ -8102,13 +8125,15 @@ "version": "1.1.15", "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-1.1.15.tgz", "integrity": "sha1-fxFLW2X6s+KjWqd1uxLw0cZJvxY=", - "dev": true + "dev": true, + "optional": true }, "smtp-connection": { "version": "2.12.0", "resolved": "https://registry.npmjs.org/smtp-connection/-/smtp-connection-2.12.0.tgz", "integrity": "sha1-1275EnyyPCJZ7bHoNJwujV4tdME=", "dev": true, + "optional": true, "requires": { "httpntlm": "1.6.1", "nodemailer-shared": "1.1.0" @@ -8127,6 +8152,7 @@ "resolved": "https://registry.npmjs.org/socks/-/socks-1.1.10.tgz", "integrity": "sha1-W4t/x8jzQcU+0FbpKbe/Tei6e1o=", "dev": true, + "optional": true, "requires": { "ip": "^1.1.4", "smart-buffer": "^1.0.13" @@ -8137,6 +8163,7 @@ "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-3.0.1.tgz", "integrity": "sha512-ZwEDymm204mTzvdqyUqOdovVr2YRd2NYskrYrF2LXyZ9qDiMAoFESGK8CRphiO7rtbo2Y757k2Nia3x2hGtalA==", "dev": true, + "optional": true, "requires": { "agent-base": "^4.1.0", "socks": "^1.1.10" @@ -8680,7 +8707,8 @@ "version": "1.7.0", "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.7.0.tgz", "integrity": "sha1-a7rwh3UA02vjTsqlhODbn+8DUgk=", - "dev": true + "dev": true, + "optional": true }, "union-value": { "version": "0.2.4", diff --git a/package.json b/package.json index df7f61a..dd110d0 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "body-parser": "^1.18.3", "bulma": "^0.7.1", "cookie-parser": "*", + "dotenv": "^6.2.0", "ejs": "^2.6.1", "errorhandler": "^1.5.0", "express": "^4.16.3",