diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 0590b00..6e6d1b2 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -58,10 +58,10 @@ - - + + - + @@ -80,8 +80,8 @@ - - + + @@ -520,14 +520,7 @@ - - - - 1467711377493 - 1467711568699 @@ -865,11 +858,18 @@ - - - - - - - - - @@ -1335,12 +1327,20 @@ + + + + + + + + - - + + - + diff --git a/fixer.js b/fixer.js index 39ef520..9eb03dc 100644 --- a/fixer.js +++ b/fixer.js @@ -172,7 +172,7 @@ let list = [ let newArray = list.map((item) => { let u = url.parse(item.feed_url); let newUrl = item.feed_url; - if (u.host === 'www.jobsite.co.uk') { + if (u.host !== '') { newUrl = `http://52.211.105.9:6566/${querystring.escape(item.feed_url)}`; } return {count:item.count, 'feed_url' : newUrl} diff --git a/jobs-corpus.js b/jobs-corpus.js index 75fa92a..64bcf39 100644 --- a/jobs-corpus.js +++ b/jobs-corpus.js @@ -256,64 +256,8 @@ sources = [ } ]; - sources = [{'count': 100, 'feed_url': 'http://www.jobserve.com/MySearch/BAEBF3BDF82B8FEF.rss'}, { - 'count': 100, - 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D47820652' -}, {'count': 100, 'feed_url': 'http://www.jobserve.com/MySearch/6DA9769BA89834AA.rss'}, { - 'count': 100, - 'feed_url': 'http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss' -}, {'count': 100, 'feed_url': 'http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss'}, { - 'count': 100, - 'feed_url': 'http://www.JobServe.com/MySearch/C7B25D86D0844A.rss' -}, {'count': 100, 'feed_url': 'http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss'}, { - 'count': 100, - 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D21564698' -}, { - 'count': 100, - 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D21564712' -}, { - 'count': 100, - 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D21942123' -}, { - 'count': 100, - 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D33166238' -}, { - 'count': 100, - 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D34888173' -}, {'count': 100, 'feed_url': 'http://www.s1jobs.com/xml/m7dp711z2r.xml'}, { - 'count': 100, - 'feed_url': 'http://www.s1jobs.com/xml/pfvf7o7z2r.xml' -}, {'count': 100, 'feed_url': 'http://www.s1jobs.com/xml/lluqnt8z2r.xml'}, { - 'count': 100, - 'feed_url': 'http://www.s1jobs.com/xml/tu33qt8z2r.xml' -}, {'count': 100, 'feed_url': 'http://www.s1jobs.com/xml/u3btnz8z2r.xml'}, { - 'count': 100, - 'feed_url': 'http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33256062' -}, {'count': 100, 'feed_url': 'http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33450169'}, { - 'count': 100, - 'feed_url': 'http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34517029' -}, {'count': 100, 'feed_url': 'http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888105'}, { - 'count': 100, - 'feed_url': 'http://www.technojobs.co.uk/rss.php/glasgow/searchtypeand/locationScotland/sortbyrelevant/jobtypeall' -}, {'count': 100, 'feed_url': 'http://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss'}, { - 'count': 100, - 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_include%3DLondon%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26compare_resolved%3DCO_LONDON%26compare_search%3DLondon%26search_emp_mkt_cd%3DALL' -}, { - 'count': 100, - 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_include%3DGlasgow%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26compare_resolved%3DTO_G1_GLASGOW%26compare_search%3DGlasgow%26search_emp_mkt_cd%3DALL' -}, { - 'count': 100, - 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26search_emp_mkt_cd%3DALL' -}, { - 'count': 100, - 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_include%3DGermany%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26compare_resolved%3DCY_GERMANY%26compare_search%3DGermany%26search_emp_mkt_cd%3DALL' -}, {'count': 100, 'feed_url': 'http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml'}, { - 'count': 100, - 'feed_url': 'http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml' -}, {'count': 100, 'feed_url': 'http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss'}, { - 'count': 100, - 'feed_url': 'http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml' -}, {'count': 100, 'feed_url': 'http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml'}]; + sources = [{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.jobserve.com%2FMySearch%2FBAEBF3BDF82B8FEF.rss"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D47820652"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.jobserve.com%2FMySearch%2F6DA9769BA89834AA.rss"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.JobServe.com%2FMySearch%2FEDF47BEA6B31EF.rss"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.JobServe.com%2FMySearch%2F3CAD044BEF2BFA.rss"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.JobServe.com%2FMySearch%2FC7B25D86D0844A.rss"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.JobServe.com%2FMySearch%2F64A3EEF615FA4C.rss"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D21564698"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D21564712"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D21942123"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D33166238"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D34888173"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.s1jobs.com%2Fxml%2Fm7dp711z2r.xml"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.s1jobs.com%2Fxml%2Fpfvf7o7z2r.xml"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.s1jobs.com%2Fxml%2Flluqnt8z2r.xml"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.s1jobs.com%2Fxml%2Ftu33qt8z2r.xml"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.s1jobs.com%2Fxml%2Fu3btnz8z2r.xml"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.purelyit.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D33256062"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.purelyit.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D33450169"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.purelyit.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D34517029"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.purelyit.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D34888105"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.technojobs.co.uk%2Frss.php%2Fglasgow%2Fsearchtypeand%2FlocationScotland%2Fsortbyrelevant%2Fjobtypeall"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.jobserve.com%2FMySearch%2F6FC7E9ED5F042ECB.rss"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_include%3DLondon%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26compare_resolved%3DCO_LONDON%26compare_search%3DLondon%26search_emp_mkt_cd%3DALL"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_include%3DGlasgow%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26compare_resolved%3DTO_G1_GLASGOW%26compare_search%3DGlasgow%26search_emp_mkt_cd%3DALL"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26search_emp_mkt_cd%3DALL"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_include%3DGermany%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26compare_resolved%3DCY_GERMANY%26compare_search%3DGermany%26search_emp_mkt_cd%3DALL"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.s1jobs.com%2Fxml%2Fb1d7e6c3a9a11964z3r.xml"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.s1jobs.com%2Fxml%2Fb1d7e6c3a9a11964z3r.xml"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.jobserve.com%2FMySearch%2FCA49421A86CA3F74.rss"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.s1jobs.com%2Fxml%2Fddeded091b6f6d33z3r.xml"},{"count":100,"feed_url":"http://52.211.105.9:6566/http%3A%2F%2Fwww.s1jobs.com%2Fxml%2F3eafc1ea20f1ca02z3r.xml"}] + ; // Pull feeds from config files: