diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 92e1d3c..20c9b49 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,7 +2,9 @@ + + @@ -25,38 +27,20 @@ - - + + - - + + - - + + - - - - - - - - - - - - - - - - - - - - + + @@ -73,8 +57,26 @@ - - + + + + + + + + + + + + + + + + + + + + @@ -83,7 +85,7 @@ - + @@ -92,7 +94,7 @@ - + @@ -127,7 +129,6 @@ - logg 6DA9769BA89834AA es6 nodejs @@ -156,6 +157,8 @@ "feed_url": " daimanuel plan + /http%3A%2F%2Fwww.jobsite + jobsite @@ -164,6 +167,7 @@ http https%3A%2F%2Fwww.technojobs "feed_url": "http://45.33.114.116:8080/ + /https%3A%2F%2Fwww.jobsite C:\dev\md\Rinser @@ -219,13 +223,13 @@ @@ -278,8 +282,8 @@ DEFINITION_ORDER - @@ -354,6 +358,10 @@ + + + + @@ -361,10 +369,6 @@ - - - - - - + - - + + - - + + @@ -1060,7 +1064,7 @@ file://$PROJECT_DIR$/gather.js - 47 + 70 @@ -1075,20 +1079,6 @@ - - - - - - - - - - - - - - @@ -1117,13 +1107,6 @@ - - - - - - - @@ -1315,13 +1298,6 @@ - - - - - - - @@ -1374,21 +1350,49 @@ - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + diff --git a/feeds/jobs-special.json b/feeds/jobs-special.json index aa4e71c..3318cbd 100644 --- a/feeds/jobs-special.json +++ b/feeds/jobs-special.json @@ -14,10 +14,7 @@ "site_url": "http://pipes.silvrtree.co.uk/jobs-special.xml" }, "sources": [ - { - "count": 100, - "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D47820652" - }, + { "count": 100, "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobserve.com%2FMySearch%2F6DA9769BA89834AA.rss" @@ -38,26 +35,6 @@ "count": 100, "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobserve.com%2FMySearch%2F64A3EEF615FA4C.rss" }, - { - "count": 100, - "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D21564698" - }, - { - "count": 100, - "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D21564712" - }, - { - "count": 100, - "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D21942123" - }, - { - "count": 100, - "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D33166238" - }, - { - "count": 100, - "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D34888173" - }, { "count": 100, "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.s1jobs.com%2Fxml%2Fm7dp711z2r.xml" @@ -102,26 +79,6 @@ "count": 100, "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobserve.com%2FMySearch%2F6FC7E9ED5F042ECB.rss" }, - { - "count": 100, - "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2C%2520asp%2C%2520web%2C%2520sql%2C%2520delphi%2C%2520vb%2C%2520vbscript%2C%2520php%2C%2520ajax%2C%2520mysql%2C%2520sqlserver%2C%2520javascript%2C%2520intranet%2C%2520vmware%2C%2520virtulization%26location_include%3DAbu%2520Dhabi%26compare_resolved%3DRE_ABUDHABI_UNITEDARABEMIRATES%26compare_search%3DAbu%2520Dhabi%26jobtype%3DX%26search_emp_mkt_cd%3DALL" - }, - { - "count": 100, - "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_include%3DLondon%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26compare_resolved%3DCO_LONDON%26compare_search%3DLondon%26search_emp_mkt_cd%3DALL" - }, - { - "count": 100, - "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_include%3DGlasgow%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26compare_resolved%3DTO_G1_GLASGOW%26compare_search%3DGlasgow%26search_emp_mkt_cd%3DALL" - }, - { - "count": 100, - "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26search_emp_mkt_cd%3DALL" - }, - { - "count": 100, - "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_include%3DGermany%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26compare_resolved%3DCY_GERMANY%26compare_search%3DGermany%26search_emp_mkt_cd%3DALL" - }, { "count": 100, "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.s1jobs.com%2Fxml%2Fb1d7e6c3a9a11964z3r.xml" diff --git a/feeds/jobs-special.json.old b/feeds/jobs-special.json.old new file mode 100644 index 0000000..80ddfd2 --- /dev/null +++ b/feeds/jobs-special.json.old @@ -0,0 +1,142 @@ +{ + "feed_name": "feed", + "default_count": 1, + "no_cdata_fields": [], + "plugins": [ + "filter_location", + "filter_reject", + "filter_md_jobs", + "filter_today_only" + ], + "meta": { + "title": "Jobs", + "description": "Combined Jobs Feed", + "site_url": "http://pipes.silvrtree.co.uk/jobs-special.xml" + }, + "sources": [ + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D47820652" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobserve.com%2FMySearch%2F6DA9769BA89834AA.rss" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobserve.com%2FMySearch%2FEDF47BEA6B31EF.rss" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobserve.com%2FMySearch%2F3CAD044BEF2BFA.rss" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobserve.com%2FMySearch%2FC7B25D86D0844A.rss" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobserve.com%2FMySearch%2F64A3EEF615FA4C.rss" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D21564698" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D21564712" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D21942123" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D33166238" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D34888173" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.s1jobs.com%2Fxml%2Fm7dp711z2r.xml" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.s1jobs.com%2Fxml%2Fpfvf7o7z2r.xml" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.s1jobs.com%2Fxml%2Flluqnt8z2r.xml" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.s1jobs.com%2Fxml%2Ftu33qt8z2r.xml" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.s1jobs.com%2Fxml%2Fu3btnz8z2r.xml" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.purelyit.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D33256062" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.purelyit.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D33450169" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.purelyit.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D34517029" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.purelyit.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D34888105" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.technojobs.co.uk%2Frss.php%2Fglasgow%2Fsearchtypeand%2FlocationScotland%2Fsortbyrelevant%2Fjobtypeall" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobserve.com%2FMySearch%2F6FC7E9ED5F042ECB.rss" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2C%2520asp%2C%2520web%2C%2520sql%2C%2520delphi%2C%2520vb%2C%2520vbscript%2C%2520php%2C%2520ajax%2C%2520mysql%2C%2520sqlserver%2C%2520javascript%2C%2520intranet%2C%2520vmware%2C%2520virtulization%26location_include%3DAbu%2520Dhabi%26compare_resolved%3DRE_ABUDHABI_UNITEDARABEMIRATES%26compare_search%3DAbu%2520Dhabi%26jobtype%3DX%26search_emp_mkt_cd%3DALL" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_include%3DLondon%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26compare_resolved%3DCO_LONDON%26compare_search%3DLondon%26search_emp_mkt_cd%3DALL" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_include%3DGlasgow%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26compare_resolved%3DTO_G1_GLASGOW%26compare_search%3DGlasgow%26search_emp_mkt_cd%3DALL" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26search_emp_mkt_cd%3DALL" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_include%3DGermany%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26compare_resolved%3DCY_GERMANY%26compare_search%3DGermany%26search_emp_mkt_cd%3DALL" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.s1jobs.com%2Fxml%2Fb1d7e6c3a9a11964z3r.xml" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/https%3A%2F%2Fwww.jobserve.com%2FMySearch%2FCA49421A86CA3F74.rss" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.s1jobs.com%2Fxml%2Fddeded091b6f6d33z3r.xml" + }, + { + "count": 100, + "feed_url": "http://45.33.114.116:8080/http%3A%2F%2Fwww.s1jobs.com%2Fxml%2F3eafc1ea20f1ca02z3r.xml" + } + ] +} diff --git a/gather.js b/gather.js index 686b22e..2b7cbe8 100644 --- a/gather.js +++ b/gather.js @@ -6,6 +6,9 @@ const read = require('fs').readFileSync; const join = require('path').join; const cron = require('node-cron'); +const util = require('util') +const fs_writeFile = util.promisify(fs.writeFile) + const str = read( join(__dirname, '/templates/rss.ejs'), 'utf8'); @@ -43,26 +46,21 @@ const rss_braider = RssBraider.createClient(braider_options); //rss_braider.logger.level('off'); function gatherV2(feedName, fileName, mode) { - const modeSuffix = {'rss': '.xml', 'json': '.json'}; return new Promise((resolve, reject) => { - logger.info(`Gathering ${feedName}...`); - rss_braider.processFeed(feedName, mode, function (err, data) { - if (err) { - logger.error(err); - return reject(err); - } + const modeSuffix = {'rss': '.xml', 'json': '.json'}; + logger.info(`Gathering ${feedName} as ${modeSuffix[mode]}...`); + rss_braider.processFeed(feedName, mode, (err, data) => { + if (err) { + logger.error(err); + return reject(err); + } - fs.writeFile(`${__dirname}/dist/${fileName}${modeSuffix[mode]}`, - data, - function (err) { - if (err) { - logger.error(err); - return reject(err); - } + fs_writeFile(`${__dirname}/dist/${fileName}${modeSuffix[mode]}`, data).then(function(){ + return resolve(`${feedName} saved`); + }); + + }); - return resolve(`${feedName} saved`); - }); - }); }); } @@ -129,7 +127,7 @@ async function gatherXML(feedName, fileName) { async function main() { await gatherV2('jobsSpecial', 'jobs-special', 'rss').then((d) => { - logger.info(d); + logger.debug(d); }).catch((e) => { logger.error(e); }); @@ -172,4 +170,4 @@ cron.schedule('3 * * * *', function () { return -1; }); -main(); +// main();