diff --git a/.idea/workspace.xml b/.idea/workspace.xml index d93de4e..0590b00 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -1,26 +1,10 @@ - - - + - - - - - - - - - - - - - - - - + + @@ -71,23 +55,23 @@ - + - - + + - + - + - - + + @@ -96,8 +80,8 @@ - - + + @@ -173,6 +157,7 @@ address /* "name" : "JobServe",*/ news + jobis @@ -236,9 +221,9 @@ @@ -535,14 +520,7 @@ - - - - 1465997901541 - 1467711377493 @@ -880,11 +858,18 @@ - \ No newline at end of file diff --git a/fixer.js b/fixer.js index 8b4dae1..39ef520 100644 --- a/fixer.js +++ b/fixer.js @@ -1,190 +1,168 @@ const querystring = require('querystring'); +const url = require('url'); let list = [ + { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss" + "count": 100, + "feed_url": "http://www.jobserve.com/MySearch/BAEBF3BDF82B8FEF.rss", + "page": "https://www.jobserve.com/gb/en/JobSearch.aspx?shid=A7ACEE7915E274717C" }, { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss" + + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=47820652" + }, + + { + + "count": 100, + "feed_url": "http://www.jobserve.com/MySearch/6DA9769BA89834AA.rss" }, { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.JobServe.com/MySearch/C7B25D86D0844A.rss" + + "count": 100, + "feed_url": "http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss" }, { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss" + + "count": 100, + "feed_url": "http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss" }, { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564698" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564712" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21942123" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33166238" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888173" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/m7dp711z2r.xml" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/pfvf7o7z2r.xml" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/lluqnt8z2r.xml" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/tu33qt8z2r.xml" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/u3btnz8z2r.xml" + + "count": 100, + "feed_url": "http://www.JobServe.com/MySearch/C7B25D86D0844A.rss" }, { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33256062" + + "count": 100, + "feed_url": "http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss" }, { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33450169" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34517029" + + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564698" }, { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888105" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.technojobs.co.uk/rss.php/glasgow/searchtypeand/locationScotland/sortbyrelevant/jobtypeall" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html,%20asp,%20web,%20sql,%20delphi,%20vb,%20vbscript,%20php,%20ajax,%20mysql,%20sqlserver,%20javascript,%20intranet,%20vmware,%20virtulization&location_include=Abu%20Dhabi&compare_resolved=RE_ABUDHABI_UNITEDARABEMIRATES&compare_search=Abu%20Dhabi&jobtype=X&search_emp_mkt_cd=ALL" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=London&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CO_LONDON&compare_search=London&search_emp_mkt_cd=ALL" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Glasgow&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=TO_G1_GLASGOW&compare_search=Glasgow&search_emp_mkt_cd=ALL" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&search_emp_mkt_cd=ALL" - } , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Germany&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CY_GERMANY&compare_search=Germany&search_emp_mkt_cd=ALL" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobisjob.co.uk/rss?what=web+developer+javascript&where=glasgow&jobType=Contract" - } - , - { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobisjob.co.uk/rss?what=javascript&where=glasgow&jobType=Contract" + + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564712" }, { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobisjob.co.uk/rss?what=asp&where=glasgow&jobType=Contract%2Ftemp" + + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21942123" }, { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss" + + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33166238" }, { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml" + + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888173" }, { - /* "name" : "JobServe",*/ - "count" : 100, - "feed_url" : "http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml" + + "count": 100, + "feed_url": "http://www.s1jobs.com/xml/m7dp711z2r.xml" + }, + { + + "count": 100, + "feed_url": "http://www.s1jobs.com/xml/pfvf7o7z2r.xml" + }, + { + + "count": 100, + "feed_url": "http://www.s1jobs.com/xml/lluqnt8z2r.xml" + }, + { + + "count": 100, + "feed_url": "http://www.s1jobs.com/xml/tu33qt8z2r.xml" + }, + { + + "count": 100, + "feed_url": "http://www.s1jobs.com/xml/u3btnz8z2r.xml" + }, + { + + "count": 100, + "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33256062" + }, + { + + "count": 100, + "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33450169" + }, + { + + "count": 100, + "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34517029" + }, + { + + "count": 100, + "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888105" + }, + { + + "count": 100, + "feed_url": "http://www.technojobs.co.uk/rss.php/glasgow/searchtypeand/locationScotland/sortbyrelevant/jobtypeall" + }, + { + + "count": 100, + "feed_url": "http://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss" + }, + { + + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=London&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CO_LONDON&compare_search=London&search_emp_mkt_cd=ALL" + }, + { + + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Glasgow&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=TO_G1_GLASGOW&compare_search=Glasgow&search_emp_mkt_cd=ALL" + }, + { + + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&search_emp_mkt_cd=ALL" + }, + { + + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Germany&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CY_GERMANY&compare_search=Germany&search_emp_mkt_cd=ALL" + }, + { + + "count": 100, + "feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" + }, + { + + "count": 100, + "feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" + }, + { + + "count": 100, + "feed_url": "http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss" + }, + { + + "count": 100, + "feed_url": "http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml" + }, + { + + "count": 100, + "feed_url": "http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml" } ]; @@ -192,9 +170,12 @@ let list = [ let newArray = list.map((item) => { - let encoded = querystring.escape(item.feed_url); - console.log(encoded); - return {count:item.count, 'feed_url' : `http://52.211.105.9:6566/${encoded}`} + let u = url.parse(item.feed_url); + let newUrl = item.feed_url; + if (u.host === 'www.jobsite.co.uk') { + newUrl = `http://52.211.105.9:6566/${querystring.escape(item.feed_url)}`; + } + return {count:item.count, 'feed_url' : newUrl} }); diff --git a/jobs-corpus.js b/jobs-corpus.js index 891a8db..75fa92a 100644 --- a/jobs-corpus.js +++ b/jobs-corpus.js @@ -1,7 +1,7 @@ var outputFile = 'jobs-special', RssBraider = require('rss-braider'), fs = require( - 'fs'), ejs = require('ejs'), read = require('fs').readFileSync, join = require( - 'path').join, str = read(join(__dirname, '/templates/rss.ejs'), - 'utf8'), feeds = {}; + 'fs'), ejs = require('ejs'), read = require('fs').readFileSync, join = require( + 'path').join, str = read(join(__dirname, '/templates/rss.ejs'), + 'utf8'), feeds = {}; var RSS = require('rss'); var log4js = require('log4js'); var logger = log4js.getLogger(); @@ -14,165 +14,165 @@ let corpus = {}; let sources = [ { - "count": 100, - "feed_url": "http://www.jobserve.com/MySearch/BAEBF3BDF82B8FEF.rss", - "page": "https://www.jobserve.com/gb/en/JobSearch.aspx?shid=A7ACEE7915E274717C" + 'count': 100, + 'feed_url': 'http://www.jobserve.com/MySearch/BAEBF3BDF82B8FEF.rss', + 'page': 'https://www.jobserve.com/gb/en/JobSearch.aspx?shid=A7ACEE7915E274717C' }, { - "count": 100, - "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=47820652" + 'count': 100, + 'feed_url': 'http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=47820652' }, { - "count": 100, - "feed_url": "http://www.jobserve.com/MySearch/6DA9769BA89834AA.rss" + 'count': 100, + 'feed_url': 'http://www.jobserve.com/MySearch/6DA9769BA89834AA.rss' }, { - "count": 100, - "feed_url": "http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss" + 'count': 100, + 'feed_url': 'http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss' }, { - "count": 100, - "feed_url": "http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss" + 'count': 100, + 'feed_url': 'http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss' }, { - "count": 100, - "feed_url": "http://www.JobServe.com/MySearch/C7B25D86D0844A.rss" + 'count': 100, + 'feed_url': 'http://www.JobServe.com/MySearch/C7B25D86D0844A.rss' }, { - "count": 100, - "feed_url": "http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss" + 'count': 100, + 'feed_url': 'http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss' }, { - "count": 100, - "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564698" + 'count': 100, + 'feed_url': 'http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564698' }, { - "count": 100, - "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564712" + 'count': 100, + 'feed_url': 'http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564712' }, { - "count": 100, - "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21942123" + 'count': 100, + 'feed_url': 'http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21942123' }, { - "count": 100, - "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33166238" + 'count': 100, + 'feed_url': 'http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33166238' }, { - "count": 100, - "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888173" + 'count': 100, + 'feed_url': 'http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888173' }, { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/m7dp711z2r.xml" + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/m7dp711z2r.xml' }, { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/pfvf7o7z2r.xml" + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/pfvf7o7z2r.xml' }, { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/lluqnt8z2r.xml" + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/lluqnt8z2r.xml' }, { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/tu33qt8z2r.xml" + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/tu33qt8z2r.xml' }, { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/u3btnz8z2r.xml" + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/u3btnz8z2r.xml' }, { - "count": 100, - "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33256062" + 'count': 100, + 'feed_url': 'http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33256062' }, { - "count": 100, - "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33450169" + 'count': 100, + 'feed_url': 'http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33450169' }, { - "count": 100, - "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34517029" + 'count': 100, + 'feed_url': 'http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34517029' }, { - "count": 100, - "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888105" + 'count': 100, + 'feed_url': 'http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888105' }, { - "count": 100, - "feed_url": "http://www.technojobs.co.uk/rss.php/glasgow/searchtypeand/locationScotland/sortbyrelevant/jobtypeall" + 'count': 100, + 'feed_url': 'http://www.technojobs.co.uk/rss.php/glasgow/searchtypeand/locationScotland/sortbyrelevant/jobtypeall' }, { - "count": 100, - "feed_url": "http://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss" + 'count': 100, + 'feed_url': 'http://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss' }, { - "count": 100, - "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=London&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CO_LONDON&compare_search=London&search_emp_mkt_cd=ALL" + 'count': 100, + 'feed_url': 'http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=London&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CO_LONDON&compare_search=London&search_emp_mkt_cd=ALL' }, { - "count": 100, - "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Glasgow&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=TO_G1_GLASGOW&compare_search=Glasgow&search_emp_mkt_cd=ALL" + 'count': 100, + 'feed_url': 'http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Glasgow&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=TO_G1_GLASGOW&compare_search=Glasgow&search_emp_mkt_cd=ALL' }, { - "count": 100, - "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&search_emp_mkt_cd=ALL" + 'count': 100, + 'feed_url': 'http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&search_emp_mkt_cd=ALL' }, { - "count": 100, - "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Germany&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CY_GERMANY&compare_search=Germany&search_emp_mkt_cd=ALL" + 'count': 100, + 'feed_url': 'http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Germany&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CY_GERMANY&compare_search=Germany&search_emp_mkt_cd=ALL' }, { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml' }, { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml' }, { - "count": 100, - "feed_url": "http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss" + 'count': 100, + 'feed_url': 'http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss' }, { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml" + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml' }, { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml" + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml' } ]; @@ -181,104 +181,163 @@ sources = [ { - "count": 100, - "feed_url": "http://www.jobserve.com/MySearch/6DA9769BA89834AA.rss" + 'count': 100, + 'feed_url': 'http://www.jobserve.com/MySearch/6DA9769BA89834AA.rss' }, { - "count": 100, - "feed_url": "http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss" + 'count': 100, + 'feed_url': 'http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss' }, { - "count": 100, - "feed_url": "http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss" + 'count': 100, + 'feed_url': 'http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss' }, { - "count": 100, - "feed_url": "http://www.JobServe.com/MySearch/C7B25D86D0844A.rss" + 'count': 100, + 'feed_url': 'http://www.JobServe.com/MySearch/C7B25D86D0844A.rss' }, { - "count": 100, - "feed_url": "http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss" + 'count': 100, + 'feed_url': 'http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss' }, { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/m7dp711z2r.xml" + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/m7dp711z2r.xml' }, { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/pfvf7o7z2r.xml" + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/pfvf7o7z2r.xml' }, { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/lluqnt8z2r.xml" + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/lluqnt8z2r.xml' }, { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/tu33qt8z2r.xml" + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/tu33qt8z2r.xml' }, { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/u3btnz8z2r.xml" + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/u3btnz8z2r.xml' }, { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml' }, { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml' }, { - "count": 100, - "feed_url": "http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss" + 'count': 100, + 'feed_url': 'http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss' }, { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml" + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml' }, { - "count": 100, - "feed_url": "http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml" + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml' } ]; + sources = [{'count': 100, 'feed_url': 'http://www.jobserve.com/MySearch/BAEBF3BDF82B8FEF.rss'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D47820652' +}, {'count': 100, 'feed_url': 'http://www.jobserve.com/MySearch/6DA9769BA89834AA.rss'}, { + 'count': 100, + 'feed_url': 'http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss' +}, {'count': 100, 'feed_url': 'http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss'}, { + 'count': 100, + 'feed_url': 'http://www.JobServe.com/MySearch/C7B25D86D0844A.rss' +}, {'count': 100, 'feed_url': 'http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D21564698' +}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D21564712' +}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D21942123' +}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D33166238' +}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D34888173' +}, {'count': 100, 'feed_url': 'http://www.s1jobs.com/xml/m7dp711z2r.xml'}, { + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/pfvf7o7z2r.xml' +}, {'count': 100, 'feed_url': 'http://www.s1jobs.com/xml/lluqnt8z2r.xml'}, { + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/tu33qt8z2r.xml' +}, {'count': 100, 'feed_url': 'http://www.s1jobs.com/xml/u3btnz8z2r.xml'}, { + 'count': 100, + 'feed_url': 'http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33256062' +}, {'count': 100, 'feed_url': 'http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33450169'}, { + 'count': 100, + 'feed_url': 'http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34517029' +}, {'count': 100, 'feed_url': 'http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888105'}, { + 'count': 100, + 'feed_url': 'http://www.technojobs.co.uk/rss.php/glasgow/searchtypeand/locationScotland/sortbyrelevant/jobtypeall' +}, {'count': 100, 'feed_url': 'http://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss'}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_include%3DLondon%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26compare_resolved%3DCO_LONDON%26compare_search%3DLondon%26search_emp_mkt_cd%3DALL' +}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_include%3DGlasgow%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26compare_resolved%3DTO_G1_GLASGOW%26compare_search%3DGlasgow%26search_emp_mkt_cd%3DALL' +}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26search_emp_mkt_cd%3DALL' +}, { + 'count': 100, + 'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_include%3DGermany%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26compare_resolved%3DCY_GERMANY%26compare_search%3DGermany%26search_emp_mkt_cd%3DALL' +}, {'count': 100, 'feed_url': 'http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml'}, { + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml' +}, {'count': 100, 'feed_url': 'http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss'}, { + 'count': 100, + 'feed_url': 'http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml' +}, {'count': 100, 'feed_url': 'http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml'}]; + // Pull feeds from config files: // feeds.simple_test_feed = require("./config/feed").feed; // Or define in-line feeds.simple_test_feed = { - "feed_name": "feed", "default_count": 1, "no_cdata_fields": [], // Don't wrap these fields in CDATA tags - "plugins": [ + 'feed_name': 'feed', 'default_count': 1, 'no_cdata_fields': [], // Don't wrap these fields in CDATA tags + 'plugins': [ 'filter_location', 'filter_reject', 'filter_md_jobs', 'filter_today_only' /*'filter_location', 'filter_today_only'*/ - ], "meta": { - "title": "Jobs", - "description": "Combined Jobs Feed", + ], 'meta': { + 'title': 'Jobs', + 'description': 'Combined Jobs Feed', 'site_url': 'http://pipes.silvrtree.co.uk/jobs-special.xml' - }, "sources": sources + }, 'sources': sources }; var braider_options = { feeds: feeds, - indent: " ", - date_sort_order: "desc", // Newest first - log_level: "debug", + indent: ' ', + date_sort_order: 'desc', // Newest first + log_level: 'debug', dedupe_fields: ['link', 'guid'], - plugins_directories: [__dirname + "/plugins/"] + plugins_directories: [__dirname + '/plugins/'] }; console.log('Working..'); var rss_braider = RssBraider.createClient(braider_options); @@ -302,7 +361,7 @@ function doFeeds() { }*/ let ejsOutput = ejs.compile(str)(j); - fs.writeFile(__dirname + "/dist/" + outputFile + ".html", + fs.writeFile(__dirname + '/dist/' + outputFile + '.html', ejsOutput, function (err) { @@ -310,10 +369,10 @@ function doFeeds() { return console.log(err); } - console.log("The file was saved!"); + console.log('The file was saved!'); }); - fs.writeFile(__dirname + "/dist/" + outputFile + ".json", + fs.writeFile(__dirname + '/dist/' + outputFile + '.json', data, function (err) { if (err) { @@ -321,21 +380,21 @@ function doFeeds() { return console.log(err); } - console.log("The file was saved!"); + console.log('The file was saved!'); }); saveCorpus(); - let newfeed = new RSS({title:'Jobs Corpus'}, j.items); + let newfeed = new RSS({title: 'Jobs Corpus'}, j.items); // console.log(newfeed); - fs.writeFile(__dirname + "/dist/" + outputFile + ".xml", newfeed.xml(' '), function(err) { + fs.writeFile(__dirname + '/dist/' + outputFile + '.xml', newfeed.xml(' '), function (err) { if (err) { return console.log(err); } - console.log("The XML file was saved!"); + console.log('The XML file was saved!'); }); @@ -344,15 +403,14 @@ function doFeeds() { } - function scoreEntry(s) { let score = 0; let used = []; - for(let t of s) { + for (let t of s) { //console.log(`processing: ${t}`); if (!corpus.hasOwnProperty(t)) { - corpus[t] = {score:0, common:1}; + corpus[t] = {score: 0, common: 1}; } else { corpus[t].common++; if (used.indexOf(t) === -1) { @@ -365,6 +423,7 @@ function scoreEntry(s) { return score; } + function processJson(j) { // console.log(j); console.log('Processing...'); @@ -383,6 +442,7 @@ function processJson(j) { return j; } + function getNoneStopWords(sentence) { let common = getStopWords(); let wordArr = sentence.match(/\w+/g), @@ -391,7 +451,7 @@ function getNoneStopWords(sentence) { word, i; for (i = 0; i < common.length; i++) { - commonObj[ common[i].trim() ] = true; + commonObj[common[i].trim()] = true; } for (i = 0; i < wordArr.length; i++) { @@ -408,24 +468,21 @@ function getStopWords() { } - - function doFeedsXML() { - rss_braider.processFeed('simple_test_feed', 'rss', function(err, data) { + rss_braider.processFeed('simple_test_feed', 'rss', function (err, data) { if (err) { return console.log(err); } + console.log('Saving', __dirname + '/dist/' + outputFile + '.xml'); - console.log('Saving', __dirname + "/dist/" + outputFile + ".xml"); - - fs.writeFile(__dirname + "/dist/" + outputFile + ".xml", data, function(err) { + fs.writeFile(__dirname + '/dist/' + outputFile + '.xml', data, function (err) { if (err) { return console.log(err); } - console.log("The file was saved!"); + console.log('The file was saved!'); }); }); @@ -433,10 +490,10 @@ function doFeedsXML() { function loadCorpus() { - let fileName = __dirname + "/dist/corpus.json"; + let fileName = __dirname + '/dist/corpus.json'; console.log(`Loading ${fileName}`); - jsonfile.readFile(fileName, function(err, obj) { + jsonfile.readFile(fileName, function (err, obj) { corpus = obj; let entries = Object.keys(corpus).length; console.log(`Corpus loaded (${entries} entries)`); @@ -445,9 +502,9 @@ function loadCorpus() { } function saveCorpus() { - let fileName = __dirname + "/dist/corpus.json"; + let fileName = __dirname + '/dist/corpus.json'; console.log(`Saving ${fileName}`); - jsonfile.writeFile(fileName, corpus,function(err, obj) { + jsonfile.writeFile(fileName, corpus, function (err, obj) { console.error(err); console.log(obj); });