diff --git a/jobs-special.js b/jobs-special.js index ca894a2..e0128cc 100644 --- a/jobs-special.js +++ b/jobs-special.js @@ -3,10 +3,6 @@ var outputFile = 'jobs-special', RssBraider = require('rss-braider'), fs = requi 'path').join, str = read(join(__dirname, '/templates/rss.ejs'), 'utf8'), feeds = {}; -var log4js = require('log4js'); -var logger = log4js.getLogger(); -const stopwords = require('stopwords-en'); - // Pull feeds from config files: // feeds.simple_test_feed = require("./config/feed").feed; // Or define in-line @@ -14,7 +10,6 @@ feeds.simple_test_feed = { "feed_name": "feed", "default_count": 1, "no_cdata_fields": [], // Don't wrap these fields in CDATA tags "plugins": [ 'filter_location', 'filter_reject', 'filter_md_jobs', 'filter_today_only' - /*'filter_location', 'filter_today_only'*/ ], "meta": { "title": "Jobs", "description": "Combined Jobs Feed", @@ -22,168 +17,167 @@ feeds.simple_test_feed = { }, "sources": [ { - "count": 100, - "feed_url": "http://www.jobserve.com/MySearch/BAEBF3BDF82B8FEF.rss", - "page": "https://www.jobserve.com/gb/en/JobSearch.aspx?shid=A7ACEE7915E274717C" - } /*, - { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=47820652" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.jobserve.com/MySearch/6DA9769BA89834AA.rss" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.JobServe.com/MySearch/C7B25D86D0844A.rss" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564698" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564712" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21942123" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33166238" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888173" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.s1jobs.com/xml/m7dp711z2r.xml" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.s1jobs.com/xml/pfvf7o7z2r.xml" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.s1jobs.com/xml/lluqnt8z2r.xml" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.s1jobs.com/xml/tu33qt8z2r.xml" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.s1jobs.com/xml/u3btnz8z2r.xml" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33256062" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33450169" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34517029" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888105" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.technojobs.co.uk/rss.php/glasgow/searchtypeand/locationScotland/sortbyrelevant/jobtypeall" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss" }, { - + /* "name" : "JobServe",*/ + "count": 100, + "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html,%20asp,%20web,%20sql,%20delphi,%20vb,%20vbscript,%20php,%20ajax,%20mysql,%20sqlserver,%20javascript,%20intranet,%20vmware,%20virtulization&location_include=Abu%20Dhabi&compare_resolved=RE_ABUDHABI_UNITEDARABEMIRATES&compare_search=Abu%20Dhabi&jobtype=X&search_emp_mkt_cd=ALL" + }, + { + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=London&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CO_LONDON&compare_search=London&search_emp_mkt_cd=ALL" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Glasgow&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=TO_G1_GLASGOW&compare_search=Glasgow&search_emp_mkt_cd=ALL" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&search_emp_mkt_cd=ALL" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Germany&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CY_GERMANY&compare_search=Germany&search_emp_mkt_cd=ALL" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml" }, { - + /* "name" : "JobServe",*/ "count": 100, "feed_url": "http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml" } -*/ ] }; @@ -232,40 +226,11 @@ rss_braider.processFeed('simple_test_feed', 'json', function(err, data) { }); }); -function getNoneStopWords(sentence) { - let common = getStopWords(); - let wordArr = sentence.match(/\w+/g), - commonObj = {}, - uncommonArr = [], - word, i; - - for (i = 0; i < common.length; i++) { - commonObj[ common[i].trim() ] = true; - } - - for (i = 0; i < wordArr.length; i++) { - word = wordArr[i].trim().toLowerCase(); - if (!commonObj[word]) { - uncommonArr.push(word); - } - } - return uncommonArr; - } - - function getStopWords() { - return stopwords; - } - - rss_braider.processFeed('simple_test_feed', 'rss', function(err, data) { if (err) { return console.log(err); } - - - console.log('Saving', __dirname + "/dist/" + outputFile + ".xml"); - fs.writeFile(__dirname + "/dist/" + outputFile + ".xml", data, function(err) { if (err) { return console.log(err);