diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index bad84df..8d70653 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -3,7 +3,8 @@
-
+
+
@@ -39,7 +40,7 @@
-
+
@@ -75,7 +76,7 @@
-
+
@@ -85,11 +86,21 @@
-
+
+
+
+
+
+
+
+
+
+
+
-
+
@@ -178,11 +189,11 @@
-
-
+
+
@@ -259,7 +270,7 @@
-
+
@@ -308,7 +319,7 @@
-
+
@@ -501,14 +512,8 @@
-
-
-
- 1458041423312
-
-
-
- 1458041423312
+
+
1458041632129
@@ -846,37 +851,44 @@
1471862770498
-
+
+ 1471949182374
+
+
+
+ 1471949182375
+
+
-
+
-
-
+
+
-
+
+
+
-
-
@@ -946,11 +958,6 @@
-
-
-
-
-
@@ -1240,14 +1247,6 @@
-
-
-
-
-
-
-
-
@@ -1280,14 +1279,6 @@
-
-
-
-
-
-
-
-
@@ -1304,5 +1295,29 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/jobs-special.js b/jobs-special.js
index 696af58..62ab7e2 100644
--- a/jobs-special.js
+++ b/jobs-special.js
@@ -1,222 +1,205 @@
-var outputFile = 'jobs-special',
- RssBraider = require('rss-braider'),
- fs = require('fs'),
- ejs = require('ejs'),
- read = require('fs').readFileSync,
- join = require('path').join,
- str = read(join(__dirname, '/templates/rss.ejs'), 'utf8'),
- feeds = {};
+var outputFile = 'jobs-special', RssBraider = require('rss-braider'), fs = require(
+ 'fs'), ejs = require('ejs'), read = require('fs').readFileSync, join = require(
+ 'path').join, str = read(join(__dirname, '/templates/rss.ejs'),
+ 'utf8'), feeds = {};
// Pull feeds from config files:
// feeds.simple_test_feed = require("./config/feed").feed;
// Or define in-line
feeds.simple_test_feed = {
- "feed_name" : "feed",
- "default_count" : 1,
- "no_cdata_fields" : [], // Don't wrap these fields in CDATA tags
- "plugins" : ['filter_location','filter_reject','filter_md_jobs','filter_today_only' ],
- "meta" : {
- "title": "Jobs",
- "description": "Combined Jobs Feed",
- 'site_url':'http://pipes.silvrtree.co.uk/jobs-special.xml'
- },
- "sources" : [
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss"
- },
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss"
- },
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.JobServe.com/MySearch/C7B25D86D0844A.rss"
- },
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss"
- },
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564698"
- }
- ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564712"
- }
- ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21942123"
- }
- ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33166238"
- }
- ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888173"
- }
- ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.s1jobs.com/xml/m7dp711z2r.xml"
- }
- ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.s1jobs.com/xml/pfvf7o7z2r.xml"
- }
- ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.s1jobs.com/xml/lluqnt8z2r.xml"
- }
- ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.s1jobs.com/xml/tu33qt8z2r.xml"
- }
- ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.s1jobs.com/xml/u3btnz8z2r.xml"
- },
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33256062"
- },
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33450169"
- }
- ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34517029"
- },
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888105"
- }
- ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.technojobs.co.uk/rss.php/glasgow/searchtypeand/locationScotland/sortbyrelevant/jobtypeall"
- }
- ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss"
- }
- ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html,%20asp,%20web,%20sql,%20delphi,%20vb,%20vbscript,%20php,%20ajax,%20mysql,%20sqlserver,%20javascript,%20intranet,%20vmware,%20virtulization&location_include=Abu%20Dhabi&compare_resolved=RE_ABUDHABI_UNITEDARABEMIRATES&compare_search=Abu%20Dhabi&jobtype=X&search_emp_mkt_cd=ALL"
- }
- ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=London&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CO_LONDON&compare_search=London&search_emp_mkt_cd=ALL"
- }
- ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Glasgow&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=TO_G1_GLASGOW&compare_search=Glasgow&search_emp_mkt_cd=ALL"
- }
- ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&search_emp_mkt_cd=ALL"
- } ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Germany&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CY_GERMANY&compare_search=Germany&search_emp_mkt_cd=ALL"
- }
- ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml"
- }
- ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml"
- }
- ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.jobisjob.co.uk/rss?what=web+developer+javascript&where=glasgow&jobType=Contract"
- }
- ,
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.jobisjob.co.uk/rss?what=javascript&where=glasgow&jobType=Contract"
- },
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.jobisjob.co.uk/rss?what=asp&where=glasgow&jobType=Contract%2Ftemp"
- },
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss"
- },
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml"
- },
- {
- /* "name" : "JobServe",*/
- "count" : 100,
- "feed_url" : "http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml"
- }
+ "feed_name": "feed", "default_count": 1, "no_cdata_fields": [], // Don't wrap these fields in CDATA tags
+ "plugins": [
+ 'filter_location',
+ 'filter_reject',
+ 'filter_md_jobs',
+ 'filter_today_only'
+ ], "meta": {
+ "title": "Jobs",
+ "description": "Combined Jobs Feed",
+ 'site_url': 'http://pipes.silvrtree.co.uk/jobs-special.xml'
+ }, "sources": [
- ]
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.jobserve.com/MySearch/6DA9769BA89834AA.rss"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.JobServe.com/MySearch/C7B25D86D0844A.rss"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564698"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564712"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21942123"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33166238"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888173"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.s1jobs.com/xml/m7dp711z2r.xml"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.s1jobs.com/xml/pfvf7o7z2r.xml"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.s1jobs.com/xml/lluqnt8z2r.xml"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.s1jobs.com/xml/tu33qt8z2r.xml"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.s1jobs.com/xml/u3btnz8z2r.xml"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33256062"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33450169"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34517029"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888105"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.technojobs.co.uk/rss.php/glasgow/searchtypeand/locationScotland/sortbyrelevant/jobtypeall"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html,%20asp,%20web,%20sql,%20delphi,%20vb,%20vbscript,%20php,%20ajax,%20mysql,%20sqlserver,%20javascript,%20intranet,%20vmware,%20virtulization&location_include=Abu%20Dhabi&compare_resolved=RE_ABUDHABI_UNITEDARABEMIRATES&compare_search=Abu%20Dhabi&jobtype=X&search_emp_mkt_cd=ALL"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=London&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CO_LONDON&compare_search=London&search_emp_mkt_cd=ALL"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Glasgow&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=TO_G1_GLASGOW&compare_search=Glasgow&search_emp_mkt_cd=ALL"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&search_emp_mkt_cd=ALL"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Germany&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CY_GERMANY&compare_search=Germany&search_emp_mkt_cd=ALL"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.jobisjob.co.uk/rss?what=web+developer+javascript&where=glasgow&jobType=Contract"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.jobisjob.co.uk/rss?what=javascript&where=glasgow&jobType=Contract"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.jobisjob.co.uk/rss?what=asp&where=glasgow&jobType=Contract%2Ftemp"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml"
+ },
+ {
+ /* "name" : "JobServe",*/
+ "count": 100,
+ "feed_url": "http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml"
+ }
+
+ ]
};
var braider_options = {
- feeds: feeds,
- indent: " ",
- date_sort_order: "desc", // Newest first
- log_level: "warn",
- dedupe_fields: ['link', 'guid'],
- plugins_directories : [__dirname + "/plugins/"]
+ feeds: feeds,
+ indent: " ",
+ date_sort_order: "desc", // Newest first
+ log_level: "warn",
+ dedupe_fields: ['link', 'guid'],
+ plugins_directories: [__dirname + "/plugins/"]
};
console.log('Working..');
var rss_braider = RssBraider.createClient(braider_options);
@@ -224,43 +207,47 @@ var rss_braider = RssBraider.createClient(braider_options);
// Override logging level (debug, info, warn, err, off)
rss_braider.logger.level('error');
-rss_braider.processFeed('simple_test_feed', 'json', function (err, data) {
- if (err) {
+rss_braider.processFeed('simple_test_feed', 'json', function(err, data) {
+ if (err) {
+ return console.log(err);
+ }
+
+ var j = JSON.parse(data);
+ var ejsOutput = ejs.compile(str)(j);
+
+ fs.writeFile(__dirname + "/dist/" + outputFile + ".html",
+ ejsOutput,
+ function(err) {
+
+ if (err) {
return console.log(err);
- }
+ }
- var j = JSON.parse(data);
- var ejsOutput = ejs.compile(str)(j);
-
- fs.writeFile(__dirname + "/dist/" + outputFile + ".html", ejsOutput, function (err) {
-
- if (err) {
- return console.log(err);
- }
-
- console.log("The file was saved!");
+ console.log("The file was saved!");
});
- fs.writeFile(__dirname + "/dist/" + outputFile + ".json", data, function (err) {
- if (err) {
- logger.error(err);
- return console.log(err);
- }
-
- console.log("The file was saved!");
- });
-});
-
-rss_braider.processFeed('simple_test_feed', 'rss', function (err, data) {
- if (err) {
+ fs.writeFile(__dirname + "/dist/" + outputFile + ".json",
+ data,
+ function(err) {
+ if (err) {
+ logger.error(err);
return console.log(err);
- }
- console.log('Saving', __dirname + "/dist/" + outputFile + ".xml");
- fs.writeFile(__dirname + "/dist/" + outputFile + ".xml", data, function (err) {
- if (err) {
- return console.log(err);
- }
+ }
- console.log("The file was saved!");
+ console.log("The file was saved!");
});
});
+
+rss_braider.processFeed('simple_test_feed', 'rss', function(err, data) {
+ if (err) {
+ return console.log(err);
+ }
+ console.log('Saving', __dirname + "/dist/" + outputFile + ".xml");
+ fs.writeFile(__dirname + "/dist/" + outputFile + ".xml", data, function(err) {
+ if (err) {
+ return console.log(err);
+ }
+
+ console.log("The file was saved!");
+ });
+});
diff --git a/plugins/filter_md_jobs.js b/plugins/filter_md_jobs.js
index 3cc10b3..6280dba 100644
--- a/plugins/filter_md_jobs.js
+++ b/plugins/filter_md_jobs.js
@@ -1,5 +1,5 @@
module.exports = function (item, itemOptions, source) {
- var patt = /(full stack|html|html5|css|javascript|sql|node|backbone|git|gulp|jquery|express|£\dk|Data Warehouse Developer)\W/ig;
+ var patt = /(full stack|html|html5|css|javascript|sql|node|backbone|git|gulp|jquery|express|£\dk|Data Warehouse Developer|iot|internet of things)\W/ig;
var result = patt.test(itemOptions.description);
var resultB = patt.test(itemOptions.title);
return (result||resultB === true) ? itemOptions : -1 ;