From e43035630671fc58bcb14819eb15b767fc4c3861 Mon Sep 17 00:00:00 2001 From: Martin Donnelly Date: Sun, 1 Oct 2017 18:25:17 +0100 Subject: [PATCH] all feeds using stone --- .idea/workspace.xml | 268 ++++++++++++----------- fixer.js | 520 +++++++++++++++++--------------------------- lifestyle.js | 339 +++++++---------------------- news.js | 188 ++++++---------- pipes-server.js | 18 +- 5 files changed, 498 insertions(+), 835 deletions(-) diff --git a/.idea/workspace.xml b/.idea/workspace.xml index b0fb2fe..92c0243 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -3,8 +3,10 @@ - - + + + + @@ -25,53 +27,43 @@ - + - + - - - - - - - - - - - - - - - - - - - - - + - - + + - + + + + + + + + + + + - - + + @@ -80,18 +72,8 @@ - - - - - - - - - - - - + + @@ -112,8 +94,28 @@ - - + + + + + + + + + + + + + + + + + + + + + + @@ -122,8 +124,8 @@ - - + + @@ -159,9 +161,11 @@ news jobis jobisjob + http://52.211.105.9:6566/ + http://stone-dev2.eu-west-1.elasticbeanstalk.com/ C:\dev\md\Rinser @@ -175,7 +179,6 @@ @@ -247,7 +251,6 @@ DEFINITION_ORDER - - - + - - - + + + + - + - @@ -907,20 +912,25 @@ - - + + - - - - + + + + + + - - + + + + + - + @@ -1031,13 +1041,6 @@ - - - - - - - @@ -1067,13 +1070,6 @@ - - - - - - - @@ -1153,13 +1149,6 @@ - - - - - - - @@ -1224,17 +1213,6 @@ - - - - - - - - - - - @@ -1281,14 +1259,6 @@ - - - - - - - - @@ -1304,47 +1274,89 @@ - + - - + + + + + + + + + + + + - - + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - + + - + + + + + + + + + \ No newline at end of file diff --git a/fixer.js b/fixer.js index 4821f30..6fece19 100644 --- a/fixer.js +++ b/fixer.js @@ -2,357 +2,245 @@ const querystring = require('querystring'); const url = require('url'); let list = [ + { - { - "count": 100, - "feed_url": "http://www.jobserve.com/MySearch/BAEBF3BDF82B8FEF.rss", - "page": "https://www.jobserve.com/gb/en/JobSearch.aspx?shid=A7ACEE7915E274717C" - }, - { + "count": 100, + "feed_url": "http://feeds2.feedburner.com/acquire" + }, - "count": 100, - "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=47820652" - }, + { - { + "count": 100, + "feed_url": "http://www.askmen.com/style.rss" + }, + { - "count": 100, - "feed_url": "http://www.jobserve.com/MySearch/6DA9769BA89834AA.rss" - }, - { + "count": 100, + "feed_url": "http://www.askmen.com/fine_living.rss" + }, + { - "count": 100, - "feed_url": "http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss" - }, - { + "count": 100, + "feed_url": "http://www.askmen.com/money.rss" + } + /*, + { - "count": 100, - "feed_url": "http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss" - }, - { + "count" : 100, + "feed_url" : "http://abetterguy.com/feed/" + }*/, - "count": 100, - "feed_url": "http://www.JobServe.com/MySearch/C7B25D86D0844A.rss" - }, - { + { - "count": 100, - "feed_url": "http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss" - }, - { + "count": 100, + "feed_url": "http://www.dmarge.com/feed" + }, + { - "count": 100, - "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564698" - }, - { + "count": 100, + "feed_url": "http://www.esquire.com/rss/all.xml" + }, + { - "count": 100, - "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564712" - }, - { + "count": 100, + "feed_url": "http://feeds.feedburner.com/gearpatrol" + }, - "count": 100, - "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21942123" - }, - { + /*{ - "count": 100, - "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33166238" - }, - { + "count": 100, + "feed_url": "http://www.getkempt.com/rss.xml" + },*/ + { - "count": 100, - "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888173" - }, - { + "count": 100, + "feed_url": "http://feeds.feedburner.com/kinowear" + }, + { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/m7dp711z2r.xml" - }, - { + "count": 100, + "feed_url": "http://www.mensfitness.co.uk/feeds/all" + }, + { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/pfvf7o7z2r.xml" - }, - { + "count": 100, + "feed_url": "http://feeds.feedburner.com/TheArtOfManliness" + }, + { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/lluqnt8z2r.xml" - }, - { + "count": 100, + "feed_url": "http://www.trunkclub.com/blog/feed" + }, + { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/tu33qt8z2r.xml" - }, - { + "count": 100, + "feed_url": "http://feeds.feedburner.com/acontinuouslean/" + }, + { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/u3btnz8z2r.xml" - }, - { + "count": 100, + "feed_url": "http://www.fashionbeans.com/feed/" + }, + { - "count": 100, - "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33256062" - }, - { + "count": 100, + "feed_url": "http://feeds.feedburner.com/MensFlair?format=xml" + }, + { - "count": 100, - "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33450169" - }, - { + "count": 100, + "feed_url": "http://feeds.feedburner.com/TheChoosyBeggar" + }, + { - "count": 100, - "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34517029" - }, - { + "count": 100, + "feed_url": "http://www.valetmag.com/distribution/rss_all.xml" + }, + { - "count": 100, - "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888105" - }, - { + "count": 100, + "feed_url": "http://putthison.com/rss" + }, + { - "count": 100, - "feed_url": "http://www.technojobs.co.uk/rss.php/glasgow/searchtypeand/locationScotland/sortbyrelevant/jobtypeall" - }, - { + "count": 100, + "feed_url": "http://dieworkwear.com/rss" + }, + { - "count": 100, - "feed_url": "http://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss" - }, - { + "count": 100, + "feed_url": "http://menofhabit.tumblr.com/rss" + }, + { - "count": 100, - "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=London&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CO_LONDON&compare_search=London&search_emp_mkt_cd=ALL" - }, - { + "count": 100, + "feed_url": "http://www.hodinkee.com/blog/rss.xml" + }, + { - "count": 100, - "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Glasgow&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=TO_G1_GLASGOW&compare_search=Glasgow&search_emp_mkt_cd=ALL" - }, - { + "count": 100, + "feed_url": "http://feeds.feedburner.com/firstinwatches" + }, + { - "count": 100, - "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&search_emp_mkt_cd=ALL" - }, - { + "count": 100, + "feed_url": "http://feeds2.feedburner.com/primermagazine/otzU" + }, + { - "count": 100, - "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Germany&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CY_GERMANY&compare_search=Germany&search_emp_mkt_cd=ALL" - }, - { + "count": 100, + "feed_url": "http://chrisguillebeau.com/3x5/feed/" + }, + { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" - }, - { + "count": 100, + "feed_url": "http://feeds.feedburner.com/TheAwesomer" + }, + { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" - }, - { + "count": 100, + "feed_url": "http://bondclothes.blogspot.com/feeds/posts/default" + }, + { - "count": 100, - "feed_url": "http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss" - }, - { + "count": 100, + "feed_url": "http://affordablebond007.tumblr.com/rss" + }, + { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml" - }, - { + "count": 100, + "feed_url": "http://thecozyspace.tumblr.com/rss" + }, + { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml" - } + "count": 100, + "feed_url": "http://www.gentlemansgazette.com/feed/" + }, + { - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.JobServe.com/MySearch/C7B25D86D0844A.rss" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564698" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564712" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21942123" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33166238" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888173" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/m7dp711z2r.xml" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/pfvf7o7z2r.xml" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/lluqnt8z2r.xml" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/tu33qt8z2r.xml" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/u3btnz8z2r.xml" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33256062" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33450169" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34517029" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888105" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.technojobs.co.uk/rss.php/glasgow/searchtypeand/locationScotland/sortbyrelevant/jobtypeall" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html,%20asp,%20web,%20sql,%20delphi,%20vb,%20vbscript,%20php,%20ajax,%20mysql,%20sqlserver,%20javascript,%20intranet,%20vmware,%20virtulization&location_include=Abu%20Dhabi&compare_resolved=RE_ABUDHABI_UNITEDARABEMIRATES&compare_search=Abu%20Dhabi&jobtype=X&search_emp_mkt_cd=ALL" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=London&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CO_LONDON&compare_search=London&search_emp_mkt_cd=ALL" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Glasgow&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=TO_G1_GLASGOW&compare_search=Glasgow&search_emp_mkt_cd=ALL" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&search_emp_mkt_cd=ALL" - } , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Germany&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CY_GERMANY&compare_search=Germany&search_emp_mkt_cd=ALL" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobisjob.co.uk/rss?what=web+developer+javascript&where=glasgow&jobType=Contract" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobisjob.co.uk/rss?what=javascript&where=glasgow&jobType=Contract" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobisjob.co.uk/rss?what=asp&where=glasgow&jobType=Contract%2Ftemp" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml" - } + "count": 100, + "feed_url": "http://feeds.feedburner.com/wellspent" + }, + { -]; + "count": 100, + "feed_url": "http://hespokestyle.com/feed/" + }, + { + + "count": 100, + "feed_url": "http://effortlessgent.com/feed/" + }, + { + + "count": 100, + "feed_url": "https://feeds.feedburner.com/thisfits" + }, + /*{ + "count": 100, + "feed_url": "http://www.businessoffashion.com/syndication/feed" + },*/ + { + "count": 100, + "feed_url": "http://dieworkwear.com/rss" + }, + { + "count": 100, + "feed_url": "http://georgehahn.com/feed/" + }, + { + "count": 100, + "feed_url": "http://www.mensjournal.com/services/rss/gear" + } + + , + { + "count": 100, + "feed_url": "http://www.mensjournal.com/services/rss/food-drink" + } + + , + { + "count": 100, + "feed_url": "http://www.mensjournal.com/services/rss/travel" + } + + , + { + "count": 100, + "feed_url": "http://www.mensjournal.com/services/rss/adventure" + } + + , + { + "count": 100, + "feed_url": "http://www.mensjournal.com/services/rss/health-fitness" + } + + , + { + "count": 100, + "feed_url": "http://www.mensjournal.com/services/rss/style" + }, + { + "count": 100, + "feed_url": "https://feeds.feedburner.com/uncrate" + }, + { + "count": 100, + "feed_url": "https://feeds.feedburner.com/ManOfMany" + }, + { + "count": 100, + "feed_url": "http://feeds.feedburner.com/nymag/vulture" + } + + , + { + "count": 100, + "feed_url": "http://www.mantelligence.com/feed/" + } + ]; let newArray = list.map((item) => { diff --git a/lifestyle.js b/lifestyle.js index fc92a18..e3d83ee 100644 --- a/lifestyle.js +++ b/lifestyle.js @@ -7,267 +7,94 @@ var outputFile = 'lifestyle', RssBraider = require('rss-braider'), fs = require( // feeds.simple_test_feed = require("./config/feed").feed; // Or define in-line feeds.simple_test_feed = { - "feed_name": "feed", "default_count": 1, "no_cdata_fields": [], // Don't wrap these fields in CDATA tags - "meta": { - "title": "Lifestyle", - "description": "Combined Lifestyle Feed", + 'feed_name': 'feed', + 'default_count': 1, + 'no_cdata_fields': [], // Don't wrap these fields in CDATA tags + 'meta': { + 'title': 'Lifestyle', + 'description': 'Combined Lifestyle Feed', 'site_url': 'http://pipes.silvrtree.co.uk/lifestyle.xml' - }, "plugins": ['filter_last_week', 'fix_images', 'fix_scripts'], "sources": [ - { - - "count": 100, - "feed_url": "http://feeds2.feedburner.com/acquire" - }, - - { - - "count": 100, - "feed_url": "http://www.askmen.com/style.rss" - }, - { - - "count": 100, - "feed_url": "http://www.askmen.com/fine_living.rss" - }, - { - - "count": 100, - "feed_url": "http://www.askmen.com/money.rss" - } - /*, - { - - "count" : 100, - "feed_url" : "http://abetterguy.com/feed/" - }*/, - - { - - "count": 100, - "feed_url": "http://www.dmarge.com/feed" - }, - { - - "count": 100, - "feed_url": "http://www.esquire.com/rss/all.xml" - }, - { - - "count": 100, - "feed_url": "http://feeds.feedburner.com/gearpatrol" - }, - - /*{ - - "count": 100, - "feed_url": "http://www.getkempt.com/rss.xml" - },*/ - { - - "count": 100, - "feed_url": "http://feeds.feedburner.com/kinowear" - }, - { - - "count": 100, - "feed_url": "http://www.mensfitness.co.uk/feeds/all" - }, - { - - "count": 100, - "feed_url": "http://feeds.feedburner.com/TheArtOfManliness" - }, - { - - "count": 100, - "feed_url": "http://www.trunkclub.com/blog/feed" - }, - { - - "count": 100, - "feed_url": "http://feeds.feedburner.com/acontinuouslean/" - }, - { - - "count": 100, - "feed_url": "http://www.fashionbeans.com/feed/" - }, - { - - "count": 100, - "feed_url": "http://feeds.feedburner.com/MensFlair?format=xml" - }, - { - - "count": 100, - "feed_url": "http://feeds.feedburner.com/TheChoosyBeggar" - }, - { - - "count": 100, - "feed_url": "http://www.valetmag.com/distribution/rss_all.xml" - }, - { - - "count": 100, - "feed_url": "http://putthison.com/rss" - }, - { - - "count": 100, - "feed_url": "http://dieworkwear.com/rss" - }, - { - - "count": 100, - "feed_url": "http://menofhabit.tumblr.com/rss" - }, - { - - "count": 100, - "feed_url": "http://www.hodinkee.com/blog/rss.xml" - }, - { - - "count": 100, - "feed_url": "http://feeds.feedburner.com/firstinwatches" - }, - { - - "count": 100, - "feed_url": "http://feeds2.feedburner.com/primermagazine/otzU" - }, - { - - "count": 100, - "feed_url": "http://chrisguillebeau.com/3x5/feed/" - }, - { - - "count": 100, - "feed_url": "http://feeds.feedburner.com/TheAwesomer" - }, - { - - "count": 100, - "feed_url": "http://bondclothes.blogspot.com/feeds/posts/default" - }, - { - - "count": 100, - "feed_url": "http://affordablebond007.tumblr.com/rss" - }, - { - - "count": 100, - "feed_url": "http://thecozyspace.tumblr.com/rss" - }, - { - - "count": 100, - "feed_url": "http://www.gentlemansgazette.com/feed/" - }, - { - - "count": 100, - "feed_url": "http://feeds.feedburner.com/wellspent" - }, - { - - "count": 100, - "feed_url": "http://hespokestyle.com/feed/" - }, - { - - "count": 100, - "feed_url": "http://effortlessgent.com/feed/" - }, - { - - "count": 100, - "feed_url": "https://feeds.feedburner.com/thisfits" - }, - /*{ - "count": 100, - "feed_url": "http://www.businessoffashion.com/syndication/feed" - },*/ - { - "count": 100, - "feed_url": "http://dieworkwear.com/rss" - }, - { - "count": 100, - "feed_url": "http://georgehahn.com/feed/" - }, - { - "count": 100, - "feed_url": "http://www.mensjournal.com/services/rss/gear" - } - - , - { - "count": 100, - "feed_url": "http://www.mensjournal.com/services/rss/food-drink" - } - - , - { - "count": 100, - "feed_url": "http://www.mensjournal.com/services/rss/travel" - } - - , - { - "count": 100, - "feed_url": "http://www.mensjournal.com/services/rss/adventure" - } - - , - { - "count": 100, - "feed_url": "http://www.mensjournal.com/services/rss/health-fitness" - } - - , - { - "count": 100, - "feed_url": "http://www.mensjournal.com/services/rss/style" - }, - { - "count": 100, - "feed_url": "https://feeds.feedburner.com/uncrate" - }, - { - "count": 100, - "feed_url": "https://feeds.feedburner.com/ManOfMany" - }, - { - "count": 100, - "feed_url": "http://feeds.feedburner.com/nymag/vulture" - } - -/*, - { - "count": 100, - "feed_url": "http://www.mantelligence.com/feed/" - } -*/ - ] + }, + 'plugins': ['filter_last_week', 'fix_images', 'fix_scripts'], + 'sources': [{'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds2.feedburner.com%2Facquire'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.askmen.com%2Fstyle.rss' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.askmen.com%2Ffine_living.rss'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.askmen.com%2Fmoney.rss' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.dmarge.com%2Ffeed'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.esquire.com%2Frss%2Fall.xml' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds.feedburner.com%2Fgearpatrol'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds.feedburner.com%2Fkinowear' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.mensfitness.co.uk%2Ffeeds%2Fall'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds.feedburner.com%2FTheArtOfManliness' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.trunkclub.com%2Fblog%2Ffeed'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds.feedburner.com%2Facontinuouslean%2F' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.fashionbeans.com%2Ffeed%2F'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds.feedburner.com%2FMensFlair%3Fformat%3Dxml' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds.feedburner.com%2FTheChoosyBeggar'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.valetmag.com%2Fdistribution%2Frss_all.xml' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fputthison.com%2Frss'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fdieworkwear.com%2Frss' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fmenofhabit.tumblr.com%2Frss'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.hodinkee.com%2Fblog%2Frss.xml' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds.feedburner.com%2Ffirstinwatches'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds2.feedburner.com%2Fprimermagazine%2FotzU' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fchrisguillebeau.com%2F3x5%2Ffeed%2F'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds.feedburner.com%2FTheAwesomer' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fbondclothes.blogspot.com%2Ffeeds%2Fposts%2Fdefault'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Faffordablebond007.tumblr.com%2Frss' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fthecozyspace.tumblr.com%2Frss'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.gentlemansgazette.com%2Ffeed%2F' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds.feedburner.com%2Fwellspent'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fhespokestyle.com%2Ffeed%2F' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Feffortlessgent.com%2Ffeed%2F'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/https%3A%2F%2Ffeeds.feedburner.com%2Fthisfits' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fgeorgehahn.com%2Ffeed%2F'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.mensjournal.com%2Fservices%2Frss%2Fgear' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.mensjournal.com%2Fservices%2Frss%2Ffood-drink'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.mensjournal.com%2Fservices%2Frss%2Ftravel' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.mensjournal.com%2Fservices%2Frss%2Fadventure'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.mensjournal.com%2Fservices%2Frss%2Fhealth-fitness' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.mensjournal.com%2Fservices%2Frss%2Fstyle'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/https%3A%2F%2Ffeeds.feedburner.com%2Funcrate' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/https%3A%2F%2Ffeeds.feedburner.com%2FManOfMany'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds.feedburner.com%2Fnymag%2Fvulture' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.mantelligence.com%2Ffeed%2F'}] }; var braider_options = { feeds: feeds, - indent: " ", - date_sort_order: "desc", // Newest first - log_level: "warn", + indent: ' ', + date_sort_order: 'desc', // Newest first + log_level: 'warn', dedupe_fields: ['link', 'guid'], - plugins_directories: [__dirname + "/plugins/"] + plugins_directories: [__dirname + '/plugins/'] }; var rss_braider = RssBraider.createClient(braider_options); // Override logging level (debug, info, warn, err, off) //rss_braider.logger.level('off'); -rss_braider.processFeed('simple_test_feed', 'json', function(err, data) { +rss_braider.processFeed('simple_test_feed', 'json', function (err, data) { if (err) { return console.log(err); } @@ -275,25 +102,25 @@ rss_braider.processFeed('simple_test_feed', 'json', function(err, data) { var j = JSON.parse(data); var ejsOutput = ejs.compile(str)(j); - fs.writeFile(__dirname + "/dist/" + outputFile + ".json", + fs.writeFile(__dirname + '/dist/' + outputFile + '.json', data, - function(err) { + function (err) { if (err) { return console.log(err); } - console.log("The file was saved!"); + console.log('The file was saved!'); }); - fs.writeFile(__dirname + "/dist/" + outputFile + ".html", + fs.writeFile(__dirname + '/dist/' + outputFile + '.html', ejsOutput, - function(err) { + function (err) { if (err) { return console.log(err); } - console.log("The file was saved!"); + console.log('The file was saved!'); }); }); diff --git a/news.js b/news.js index 8bf89f9..db42d0d 100644 --- a/news.js +++ b/news.js @@ -13,142 +13,78 @@ var logger = log4js.getLogger(); // Or define in-line feeds.simple_test_feed = { - "feed_name": "feed", - "default_count": 1, - "no_cdata_fields": ['description'], // Don't wrap these fields in CDATA tags - "meta": { - "title": "News", - "description": "Combined News Feed", + 'feed_name': 'feed', + 'default_count': 1, + 'no_cdata_fields': ['description'], // Don't wrap these fields in CDATA tags + 'meta': { + 'title': 'News', + 'description': 'Combined News Feed', 'site_url': 'http://pipes.silvrtree.co.uk/news.xml' }, - "plugins": [ + 'plugins': [ 'filter_3_days', 'fix_images', 'fix_scripts' ], - 'custom_namespaces' : { - "content" : "http://purl.org/rss/1.0/modules/content/", - "slash" : "http://purl.org/rss/1.0/modules/slash/", - "media" : "http://search.yahoo.com/mrss/", - "ev" : "http://purl.org/rss/2.0/modules/event/", - "sy" : "http://purl.org/rss/1.0/modules/syndication/", - "wfw" : "http://wellformedweb.org/CommentAPI/", - "dc" : "http://purl.org/dc/elements/1.1/", - "atom" : "http://www.w3.org/2005/Atom" - }, - "sources": [ - { - "count": 100, - "feed_url": "http://qz.com/feed/" - }, - { - "count": 100, - "feed_url": "http://feeds2.feedburner.com/businessinsider" - }, - { - "count": 100, - "feed_url": "http://feeds.reuters.com/reuters/UKTopNews" - }, - { - "count": 100, - "feed_url": "http://feeds.reuters.com/reuters/UKdomesticNews" - }, - { - "count": 100, - "feed_url": "http://feeds.reuters.com/reuters/UKWorldNews" - }, - { - "count": 100, - "feed_url": "http://feeds.reuters.com/reuters/topNews" - }, - { - "count": 100, - "feed_url": "http://feeds.reuters.com/Reuters/worldNews" - } - - , - /*{ - "count": 100, - "feed_url": "http://feeds.reuters.com/reuters/globalmarketsNews" - },*/ - { - "count": 100, - "feed_url": "http://hosted2.ap.org/atom/APDEFAULT/cae69a7523db45408eeb2b3a98c0c9c5" - }, - { - "count": 100, - "feed_url": "http://www.cityam.com/feeds/main.xml" - }, - { - "count": 100, - "feed_url": "http://www.londonlovesbusiness.com/navrss?navsectioncode=121" - }, - { - "count": 100, - "feed_url": "http://www.ibtimes.co.uk/rss/uk" - }, - { - "count": 100, - "feed_url": "http://feeds.bbci.co.uk/news/rss.xml?edition=uk" - }, - { - "count": 100, - "feed_url": "http://feeds.bbci.co.uk/news/scotland/rss.xml?edition=uk" - }, - { - "count": 100, - "feed_url": "http://news.google.com/?output=rss" - }, - { - "count": 100, - "feed_url": "http://feeds.feedburner.com/TheAtlantic" - }, - { - "count": 100, - "feed_url": "http://www.telegraph.co.uk/news/rss.xml" - }, - { - "count": 100, - "feed_url": "http://www.guardian.co.uk/rssfeed/0,,1,00.xml" - }, - { - "count": 100, - "feed_url": "http://www.marketwatch.com/rss/topstories" - }, - { - "count": 100, - "feed_url": "http://feeds.feedburner.com/wingsoverscotland/FgKv" - }, - { - "count": 100, - "feed_url": "http://bellacaledonia.org.uk/feed/" - } - , - { - "count": 100, - "feed_url": "http://www.theweek.co.uk/feeds/all" - } - , - { - "count": 100, - "feed_url": "http://fivethirtyeight.com/all/feed" - } - ] + 'custom_namespaces': { + 'content': 'http://purl.org/rss/1.0/modules/content/', + 'slash': 'http://purl.org/rss/1.0/modules/slash/', + 'media': 'http://search.yahoo.com/mrss/', + 'ev': 'http://purl.org/rss/2.0/modules/event/', + 'sy': 'http://purl.org/rss/1.0/modules/syndication/', + 'wfw': 'http://wellformedweb.org/CommentAPI/', + 'dc': 'http://purl.org/dc/elements/1.1/', + 'atom': 'http://www.w3.org/2005/Atom' + }, + 'sources': [{'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fqz.com%2Ffeed%2F'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds2.feedburner.com%2Fbusinessinsider' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds.reuters.com%2Freuters%2FUKTopNews'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds.reuters.com%2Freuters%2FUKdomesticNews' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds.reuters.com%2Freuters%2FUKWorldNews'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds.reuters.com%2Freuters%2FtopNews' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds.reuters.com%2FReuters%2FworldNews'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fhosted2.ap.org%2Fatom%2FAPDEFAULT%2Fcae69a7523db45408eeb2b3a98c0c9c5' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.cityam.com%2Ffeeds%2Fmain.xml'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.londonlovesbusiness.com%2Fnavrss%3Fnavsectioncode%3D121' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.ibtimes.co.uk%2Frss%2Fuk'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds.bbci.co.uk%2Fnews%2Frss.xml%3Fedition%3Duk' + }, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds.bbci.co.uk%2Fnews%2Fscotland%2Frss.xml%3Fedition%3Duk' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fnews.google.com%2F%3Foutput%3Drss'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds.feedburner.com%2FTheAtlantic' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.telegraph.co.uk%2Fnews%2Frss.xml'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.guardian.co.uk%2Frssfeed%2F0%2C%2C1%2C00.xml' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.marketwatch.com%2Frss%2Ftopstories'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffeeds.feedburner.com%2Fwingsoverscotland%2FFgKv' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fbellacaledonia.org.uk%2Ffeed%2F'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.theweek.co.uk%2Ffeeds%2Fall' + }, {'count': 100, 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Ffivethirtyeight.com%2Fall%2Ffeed'}] } ; var braider_options = { feeds: feeds, - indent: " ", - date_sort_order: "desc", // Newest first - log_level: "warn", + indent: ' ', + date_sort_order: 'desc', // Newest first + log_level: 'warn', dedupe_fields: ['link', 'guid'], - plugins_directories: [__dirname + "/plugins/"] + plugins_directories: [__dirname + '/plugins/'] }; var rss_braider = RssBraider.createClient(braider_options); // Override logging level (debug, info, warn, err, off) //rss_braider.logger.level('off'); -rss_braider.processFeed('simple_test_feed', 'json', function(err, data) { +rss_braider.processFeed('simple_test_feed', 'json', function (err, data) { console.log('A'); if (err) { logger.error(err); @@ -163,26 +99,26 @@ rss_braider.processFeed('simple_test_feed', 'json', function(err, data) { var ejsOutput = ejs.compile(str)(j); console.log('D'); - fs.writeFile(__dirname + "/dist/" + outputFile + ".html", + fs.writeFile(__dirname + '/dist/' + outputFile + '.html', ejsOutput, - function(err) { + function (err) { if (err) { logger.error(err); return console.error(err); } - console.log("The file was saved!"); + console.log('The file was saved!'); }); - fs.writeFile(__dirname + "/dist/" + outputFile + ".json", + fs.writeFile(__dirname + '/dist/' + outputFile + '.json', data, - function(err) { + function (err) { if (err) { logger.error(err); return console.error(err); } - console.log("The file was saved!"); + console.log('The file was saved!'); }); }); diff --git a/pipes-server.js b/pipes-server.js index 673815a..8c46cbf 100644 --- a/pipes-server.js +++ b/pipes-server.js @@ -5,12 +5,12 @@ var lastcheck = 0; var btcCache = {}, fxCache = {}; //app.configure(function () { - app.set('port', 9002); +app.set('port', 9002); app.use(morgan('combined')); app.use(cookieparser('your secret here')); app.use(session({ - secret: '1234567890QWERTY', resave: false, - saveUninitialized: false + secret: '1234567890QWERTY', resave: false, + saveUninitialized: false })); /* 'default', 'short', 'tiny', 'dev' */ app.use(methodoverride()); @@ -21,12 +21,12 @@ app.use(bodyparser.urlencoded({extended: false})); app.use(bodyparser.json()); app.use(function (req, res, next) { - res.header("Access-Control-Allow-Origin", "*"); - res.header("Access-Control-Allow-Headers", "X-Requested-With"); - next(); + res.header("Access-Control-Allow-Origin", "*"); + res.header("Access-Control-Allow-Headers", "X-Requested-With"); + next(); }); - //app.use(app.router); - app.use(express.static(path.join(__dirname, 'dist'))); +//app.use(app.router); +app.use(express.static(path.join(__dirname, 'dist'))); app.use(errorhandler({dumpExceptions: true, showStack: true})); @@ -38,5 +38,5 @@ app.use(errorhandler({dumpExceptions: true, showStack: true})); * create the server */ http.createServer(app).listen(app.get('port'), function () { - console.log("Express server listening on port " + app.get('port')); + console.log("Express server listening on port " + app.get('port')); });