From f29703892ad790bb61dbb86e80de54fbf60fbcf6 Mon Sep 17 00:00:00 2001 From: Martin Donnelly Date: Wed, 24 Aug 2016 09:39:58 +0100 Subject: [PATCH] fixed plugins --- .idea/workspace.xml | 105 +++++---- jobs-special.js | 471 ++++++++++++++++++-------------------- plugins/filter_md_jobs.js | 2 +- 3 files changed, 290 insertions(+), 288 deletions(-) diff --git a/.idea/workspace.xml b/.idea/workspace.xml index bad84df..8d70653 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -3,7 +3,8 @@ - + + @@ -39,7 +40,7 @@ - + @@ -75,7 +76,7 @@ - + @@ -85,11 +86,21 @@ - + + + + + + + + + + + - + @@ -178,11 +189,11 @@ @@ -259,7 +270,7 @@ - + @@ -308,7 +319,7 @@ - + @@ -501,14 +512,8 @@ - - - - 1458041423312 - 1458041632129 @@ -846,37 +851,44 @@ - - - - + + - + + + - - @@ -946,11 +958,6 @@ - - - - - @@ -1240,14 +1247,6 @@ - - - - - - - - @@ -1280,14 +1279,6 @@ - - - - - - - - @@ -1304,5 +1295,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/jobs-special.js b/jobs-special.js index 696af58..62ab7e2 100644 --- a/jobs-special.js +++ b/jobs-special.js @@ -1,222 +1,205 @@ -var outputFile = 'jobs-special', - RssBraider = require('rss-braider'), - fs = require('fs'), - ejs = require('ejs'), - read = require('fs').readFileSync, - join = require('path').join, - str = read(join(__dirname, '/templates/rss.ejs'), 'utf8'), - feeds = {}; +var outputFile = 'jobs-special', RssBraider = require('rss-braider'), fs = require( + 'fs'), ejs = require('ejs'), read = require('fs').readFileSync, join = require( + 'path').join, str = read(join(__dirname, '/templates/rss.ejs'), + 'utf8'), feeds = {}; // Pull feeds from config files: // feeds.simple_test_feed = require("./config/feed").feed; // Or define in-line feeds.simple_test_feed = { - "feed_name" : "feed", - "default_count" : 1, - "no_cdata_fields" : [], // Don't wrap these fields in CDATA tags - "plugins" : ['filter_location','filter_reject','filter_md_jobs','filter_today_only' ], - "meta" : { - "title": "Jobs", - "description": "Combined Jobs Feed", - 'site_url':'http://pipes.silvrtree.co.uk/jobs-special.xml' - }, - "sources" : [ - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.JobServe.com/MySearch/C7B25D86D0844A.rss" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564698" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564712" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21942123" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33166238" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888173" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/m7dp711z2r.xml" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/pfvf7o7z2r.xml" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/lluqnt8z2r.xml" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/tu33qt8z2r.xml" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/u3btnz8z2r.xml" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33256062" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33450169" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34517029" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888105" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.technojobs.co.uk/rss.php/glasgow/searchtypeand/locationScotland/sortbyrelevant/jobtypeall" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html,%20asp,%20web,%20sql,%20delphi,%20vb,%20vbscript,%20php,%20ajax,%20mysql,%20sqlserver,%20javascript,%20intranet,%20vmware,%20virtulization&location_include=Abu%20Dhabi&compare_resolved=RE_ABUDHABI_UNITEDARABEMIRATES&compare_search=Abu%20Dhabi&jobtype=X&search_emp_mkt_cd=ALL" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=London&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CO_LONDON&compare_search=London&search_emp_mkt_cd=ALL" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Glasgow&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=TO_G1_GLASGOW&compare_search=Glasgow&search_emp_mkt_cd=ALL" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&search_emp_mkt_cd=ALL" - } , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Germany&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CY_GERMANY&compare_search=Germany&search_emp_mkt_cd=ALL" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobisjob.co.uk/rss?what=web+developer+javascript&where=glasgow&jobType=Contract" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobisjob.co.uk/rss?what=javascript&where=glasgow&jobType=Contract" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobisjob.co.uk/rss?what=asp&where=glasgow&jobType=Contract%2Ftemp" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml" - }, - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml" - } + "feed_name": "feed", "default_count": 1, "no_cdata_fields": [], // Don't wrap these fields in CDATA tags + "plugins": [ + 'filter_location', + 'filter_reject', + 'filter_md_jobs', + 'filter_today_only' + ], "meta": { + "title": "Jobs", + "description": "Combined Jobs Feed", + 'site_url': 'http://pipes.silvrtree.co.uk/jobs-special.xml' + }, "sources": [ - ] + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.jobserve.com/MySearch/6DA9769BA89834AA.rss" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.JobServe.com/MySearch/C7B25D86D0844A.rss" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564698" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564712" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21942123" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33166238" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888173" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.s1jobs.com/xml/m7dp711z2r.xml" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.s1jobs.com/xml/pfvf7o7z2r.xml" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.s1jobs.com/xml/lluqnt8z2r.xml" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.s1jobs.com/xml/tu33qt8z2r.xml" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.s1jobs.com/xml/u3btnz8z2r.xml" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33256062" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33450169" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34517029" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888105" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.technojobs.co.uk/rss.php/glasgow/searchtypeand/locationScotland/sortbyrelevant/jobtypeall" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html,%20asp,%20web,%20sql,%20delphi,%20vb,%20vbscript,%20php,%20ajax,%20mysql,%20sqlserver,%20javascript,%20intranet,%20vmware,%20virtulization&location_include=Abu%20Dhabi&compare_resolved=RE_ABUDHABI_UNITEDARABEMIRATES&compare_search=Abu%20Dhabi&jobtype=X&search_emp_mkt_cd=ALL" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=London&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CO_LONDON&compare_search=London&search_emp_mkt_cd=ALL" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Glasgow&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=TO_G1_GLASGOW&compare_search=Glasgow&search_emp_mkt_cd=ALL" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&search_emp_mkt_cd=ALL" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Germany&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CY_GERMANY&compare_search=Germany&search_emp_mkt_cd=ALL" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.jobisjob.co.uk/rss?what=web+developer+javascript&where=glasgow&jobType=Contract" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.jobisjob.co.uk/rss?what=javascript&where=glasgow&jobType=Contract" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.jobisjob.co.uk/rss?what=asp&where=glasgow&jobType=Contract%2Ftemp" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml" + }, + { + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml" + } + + ] }; var braider_options = { - feeds: feeds, - indent: " ", - date_sort_order: "desc", // Newest first - log_level: "warn", - dedupe_fields: ['link', 'guid'], - plugins_directories : [__dirname + "/plugins/"] + feeds: feeds, + indent: " ", + date_sort_order: "desc", // Newest first + log_level: "warn", + dedupe_fields: ['link', 'guid'], + plugins_directories: [__dirname + "/plugins/"] }; console.log('Working..'); var rss_braider = RssBraider.createClient(braider_options); @@ -224,43 +207,47 @@ var rss_braider = RssBraider.createClient(braider_options); // Override logging level (debug, info, warn, err, off) rss_braider.logger.level('error'); -rss_braider.processFeed('simple_test_feed', 'json', function (err, data) { - if (err) { +rss_braider.processFeed('simple_test_feed', 'json', function(err, data) { + if (err) { + return console.log(err); + } + + var j = JSON.parse(data); + var ejsOutput = ejs.compile(str)(j); + + fs.writeFile(__dirname + "/dist/" + outputFile + ".html", + ejsOutput, + function(err) { + + if (err) { return console.log(err); - } + } - var j = JSON.parse(data); - var ejsOutput = ejs.compile(str)(j); - - fs.writeFile(__dirname + "/dist/" + outputFile + ".html", ejsOutput, function (err) { - - if (err) { - return console.log(err); - } - - console.log("The file was saved!"); + console.log("The file was saved!"); }); - fs.writeFile(__dirname + "/dist/" + outputFile + ".json", data, function (err) { - if (err) { - logger.error(err); - return console.log(err); - } - - console.log("The file was saved!"); - }); -}); - -rss_braider.processFeed('simple_test_feed', 'rss', function (err, data) { - if (err) { + fs.writeFile(__dirname + "/dist/" + outputFile + ".json", + data, + function(err) { + if (err) { + logger.error(err); return console.log(err); - } - console.log('Saving', __dirname + "/dist/" + outputFile + ".xml"); - fs.writeFile(__dirname + "/dist/" + outputFile + ".xml", data, function (err) { - if (err) { - return console.log(err); - } + } - console.log("The file was saved!"); + console.log("The file was saved!"); }); }); + +rss_braider.processFeed('simple_test_feed', 'rss', function(err, data) { + if (err) { + return console.log(err); + } + console.log('Saving', __dirname + "/dist/" + outputFile + ".xml"); + fs.writeFile(__dirname + "/dist/" + outputFile + ".xml", data, function(err) { + if (err) { + return console.log(err); + } + + console.log("The file was saved!"); + }); +}); diff --git a/plugins/filter_md_jobs.js b/plugins/filter_md_jobs.js index 3cc10b3..6280dba 100644 --- a/plugins/filter_md_jobs.js +++ b/plugins/filter_md_jobs.js @@ -1,5 +1,5 @@ module.exports = function (item, itemOptions, source) { - var patt = /(full stack|html|html5|css|javascript|sql|node|backbone|git|gulp|jquery|express|£\dk|Data Warehouse Developer)\W/ig; + var patt = /(full stack|html|html5|css|javascript|sql|node|backbone|git|gulp|jquery|express|£\dk|Data Warehouse Developer|iot|internet of things)\W/ig; var result = patt.test(itemOptions.description); var resultB = patt.test(itemOptions.title); return (result||resultB === true) ? itemOptions : -1 ;