martin/Rinser
martin/Rinser
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

jobs fix, reverted

Browse Source
This commit is contained in:
Martin Donnelly 2017-09-11 12:29:20 +01:00
parent b61e60cd5e
commit 8c2c1d2281
3 changed files with 374 additions and 351 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

99
.idea/workspace.xml
View File

@ -1,26 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?> <?xml version="1.0" encoding="UTF-8"?>
<project version="4"> <project version="4">
<component name="ChangeListManager"> <component name="ChangeListManager">
<list default="true" id="c2fd5632-2143-4c7e-8b0d-96d7692b700b" name="Default" comment="jobs fix, reverted"> <list default="true" id="c2fd5632-2143-4c7e-8b0d-96d7692b700b" name="Default" comment="">
<change type="NEW" beforePath="" afterPath="$PROJECT_DIR$/.eslintrc" />
<change type="NEW" beforePath="" afterPath="$PROJECT_DIR$/fixer.js" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" /> <change type="MODIFICATION" beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/jobs-local.js" afterPath="$PROJECT_DIR$/jobs-local.js" /> <change type="MODIFICATION" beforePath="$PROJECT_DIR$/fixer.js" afterPath="$PROJECT_DIR$/fixer.js" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/jobs-special.js" afterPath="$PROJECT_DIR$/jobs-special.js" /> <change type="MODIFICATION" beforePath="$PROJECT_DIR$/jobs-corpus.js" afterPath="$PROJECT_DIR$/jobs-corpus.js" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/node_modules/ejs/Jakefile" afterPath="$PROJECT_DIR$/node_modules/ejs/Jakefile" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/node_modules/ejs/README.md" afterPath="$PROJECT_DIR$/node_modules/ejs/README.md" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/node_modules/ejs/ejs.js" afterPath="$PROJECT_DIR$/node_modules/ejs/ejs.js" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/node_modules/ejs/ejs.min.js" afterPath="$PROJECT_DIR$/node_modules/ejs/ejs.min.js" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/node_modules/ejs/lib/ejs.js" afterPath="$PROJECT_DIR$/node_modules/ejs/lib/ejs.js" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/node_modules/ejs/lib/utils.js" afterPath="$PROJECT_DIR$/node_modules/ejs/lib/utils.js" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/node_modules/ejs/package.json" afterPath="$PROJECT_DIR$/node_modules/ejs/package.json" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/node_modules/ejs/test/ejs.js" afterPath="$PROJECT_DIR$/node_modules/ejs/test/ejs.js" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/node_modules/ejs/test/fixtures/no.newlines.error.ejs" afterPath="$PROJECT_DIR$/node_modules/ejs/test/fixtures/no.newlines.error.ejs" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/node_modules/express/package.json" afterPath="$PROJECT_DIR$/node_modules/express/package.json" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/node_modules/lodash/package.json" afterPath="$PROJECT_DIR$/node_modules/lodash/package.json" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/node_modules/rss-braider/.npmignore" afterPath="$PROJECT_DIR$/node_modules/rss-braider/.npmignore" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/node_modules/rss-braider/README.md" afterPath="$PROJECT_DIR$/node_modules/rss-braider/README.md" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/node_modules/rss-braider/package.json" afterPath="$PROJECT_DIR$/node_modules/rss-braider/package.json" />
</list> </list>
<ignored path="rinser.iws" /> <ignored path="rinser.iws" />
<ignored path=".idea/workspace.xml" /> <ignored path=".idea/workspace.xml" />
@ -71,23 +55,23 @@
</provider> </provider>
</entry> </entry>
</file> </file>
<file leaf-file-name="jobs-corpus.js" pinned="false" current-in-tab="false"> <file leaf-file-name="jobs-corpus.js" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/jobs-corpus.js"> <entry file="file://$PROJECT_DIR$/jobs-corpus.js">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="3840"> <state relative-caret-position="244">
<caret line="256" column="2" lean-forward="true" selection-start-line="256" selection-start-column="2" selection-end-line="256" selection-end-column="2" /> <caret line="258" column="0" lean-forward="false" selection-start-line="258" selection-start-column="0" selection-end-line="258" selection-end-column="0" />
<folding> <folding>
<marker date="1500448512000" expanded="true" signature="9713:9733" ph="..." /> <marker date="1505129349000" expanded="true" signature="15038:15058" ph="..." />
</folding> </folding>
</state> </state>
</provider> </provider>
</entry> </entry>
</file> </file>
<file leaf-file-name="jobs-local.js" pinned="false" current-in-tab="true"> <file leaf-file-name="jobs-local.js" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/jobs-local.js"> <entry file="file://$PROJECT_DIR$/jobs-local.js">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="216"> <state relative-caret-position="1588">
<caret line="32" column="17" lean-forward="true" selection-start-line="32" selection-start-column="17" selection-end-line="32" selection-end-column="17" /> <caret line="119" column="0" lean-forward="true" selection-start-line="119" selection-start-column="0" selection-end-line="119" selection-end-column="0" />
<folding /> <folding />
</state> </state>
</provider> </provider>
@ -96,8 +80,8 @@
<file leaf-file-name="fixer.js" pinned="false" current-in-tab="false"> <file leaf-file-name="fixer.js" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/fixer.js"> <entry file="file://$PROJECT_DIR$/fixer.js">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="416"> <state relative-caret-position="386">
<caret line="194" column="34" lean-forward="false" selection-start-line="194" selection-start-column="34" selection-end-line="194" selection-end-column="34" /> <caret line="170" column="0" lean-forward="true" selection-start-line="170" selection-start-column="0" selection-end-line="170" selection-end-column="0" />
<folding /> <folding />
</state> </state>
</provider> </provider>
@ -173,6 +157,7 @@
<find>address</find> <find>address</find>
<find>/* &quot;name&quot; : &quot;JobServe&quot;,*/</find> <find>/* &quot;name&quot; : &quot;JobServe&quot;,*/</find>
<find>news</find> <find>news</find>
<find>jobis</find>
</findStrings> </findStrings>
<replaceStrings> <replaceStrings>
<replace /> <replace />
@ -236,9 +221,9 @@
<option value="$APPLICATION_CONFIG_DIR$/scratches/jobs-new.js" /> <option value="$APPLICATION_CONFIG_DIR$/scratches/jobs-new.js" />
<option value="$PROJECT_DIR$/.eslintrc" /> <option value="$PROJECT_DIR$/.eslintrc" />
<option value="$PROJECT_DIR$/jobs-new.js" /> <option value="$PROJECT_DIR$/jobs-new.js" />
<option value="$PROJECT_DIR$/jobs-corpus.js" />
<option value="$PROJECT_DIR$/fixer.js" />
<option value="$PROJECT_DIR$/jobs-local.js" /> <option value="$PROJECT_DIR$/jobs-local.js" />
<option value="$PROJECT_DIR$/fixer.js" />
<option value="$PROJECT_DIR$/jobs-corpus.js" />
</list> </list>
</option> </option>
</component> </component>
@ -535,14 +520,7 @@
<workItem from="1500370528349" duration="12243000" /> <workItem from="1500370528349" duration="12243000" />
<workItem from="1500567464596" duration="598000" /> <workItem from="1500567464596" duration="598000" />
<workItem from="1501109852743" duration="15000" /> <workItem from="1501109852743" duration="15000" />
<workItem from="1505124639040" duration="1435000" /> <workItem from="1505124639040" duration="3186000" />
</task>
<task id="LOCAL-00051" summary="tidied feeds a bit and fixed fit&#10;">
<created>1465997901541</created>
<option name="number" value="00051" />
<option name="presentableId" value="LOCAL-00051" />
<option name="project" value="LOCAL" />
<updated>1465997901542</updated>
</task> </task>
<task id="LOCAL-00052" summary="changing log level for bunyan"> <task id="LOCAL-00052" summary="changing log level for bunyan">
<created>1467711377493</created> <created>1467711377493</created>
@ -880,11 +858,18 @@
<option name="project" value="LOCAL" /> <option name="project" value="LOCAL" />
<updated>1500448522129</updated> <updated>1500448522129</updated>
</task> </task>
<option name="localTasksCounter" value="100" /> <task id="LOCAL-00100" summary="jobs fix, reverted">
<created>1505126085720</created>
<option name="number" value="00100" />
<option name="presentableId" value="LOCAL-00100" />
<option name="project" value="LOCAL" />
<updated>1505126085721</updated>
</task>
<option name="localTasksCounter" value="101" />
<servers /> <servers />
</component> </component>
<component name="TimeTrackingManager"> <component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="89007000" /> <option name="totallyTimeSpent" value="90758000" />
</component> </component>
<component name="ToolWindowManager"> <component name="ToolWindowManager">
<frame x="32" y="23" width="1966" height="1129" extended-state="0" /> <frame x="32" y="23" width="1966" height="1129" extended-state="0" />
@ -893,9 +878,9 @@
<window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.13419023" sideWeight="0.65890414" order="0" side_tool="false" content_ui="combo" /> <window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.13419023" sideWeight="0.65890414" order="0" side_tool="false" content_ui="combo" />
<window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" /> <window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
<window_info id="Docker" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" /> <window_info id="Docker" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.36102867" sideWeight="0.44267353" order="7" side_tool="true" content_ui="tabs" /> <window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.36102867" sideWeight="0.44473007" order="7" side_tool="true" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33333334" sideWeight="0.6411311" order="2" side_tool="false" content_ui="tabs" /> <window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33333334" sideWeight="0.6385604" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Version Control" active="true" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.36102867" sideWeight="0.5573265" order="7" side_tool="false" content_ui="tabs" /> <window_info id="Version Control" active="true" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.36102867" sideWeight="0.5552699" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" /> <window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32896176" sideWeight="0.4838976" order="7" side_tool="false" content_ui="tabs" /> <window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32896176" sideWeight="0.4838976" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.3997773" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" /> <window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.3997773" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
@ -1334,31 +1319,31 @@
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/jobs-corpus.js">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="3840">
<caret line="256" column="2" lean-forward="true" selection-start-line="256" selection-start-column="2" selection-end-line="256" selection-end-column="2" />
<folding>
<marker date="1500448512000" expanded="true" signature="9713:9733" ph="..." />
</folding>
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/fixer.js"> <entry file="file://$PROJECT_DIR$/fixer.js">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="416"> <state relative-caret-position="386">
<caret line="194" column="34" lean-forward="false" selection-start-line="194" selection-start-column="34" selection-end-line="194" selection-end-column="34" /> <caret line="170" column="0" lean-forward="true" selection-start-line="170" selection-start-column="0" selection-end-line="170" selection-end-column="0" />
<folding /> <folding />
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/jobs-local.js"> <entry file="file://$PROJECT_DIR$/jobs-local.js">
<provider selected="true" editor-type-id="text-editor"> <provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="216"> <state relative-caret-position="1588">
<caret line="32" column="17" lean-forward="true" selection-start-line="32" selection-start-column="17" selection-end-line="32" selection-end-column="17" /> <caret line="119" column="0" lean-forward="true" selection-start-line="119" selection-start-column="0" selection-end-line="119" selection-end-column="0" />
<folding /> <folding />
</state> </state>
</provider> </provider>
</entry> </entry>
<entry file="file://$PROJECT_DIR$/jobs-corpus.js">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="244">
<caret line="258" column="0" lean-forward="false" selection-start-line="258" selection-start-column="0" selection-end-line="258" selection-end-column="0" />
<folding>
<marker date="1505129349000" expanded="true" signature="15038:15058" ph="..." />
</folding>
</state>
</provider>
</entry>
</component> </component>
</project> </project>

309
fixer.js
View File

@ -1,190 +1,168 @@
const querystring = require('querystring'); const querystring = require('querystring');
const url = require('url');
let list = [ let list = [
{ {
/* "name" : "JobServe",*/ "count": 100,
"count" : 100, "feed_url": "http://www.jobserve.com/MySearch/BAEBF3BDF82B8FEF.rss",
"feed_url" : "http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss" "page": "https://www.jobserve.com/gb/en/JobSearch.aspx?shid=A7ACEE7915E274717C"
}, },
{ {
/* "name" : "JobServe",*/
"count" : 100, "count": 100,
"feed_url" : "http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss" "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=47820652"
},
{
"count": 100,
"feed_url": "http://www.jobserve.com/MySearch/6DA9769BA89834AA.rss"
}, },
{ {
/* "name" : "JobServe",*/
"count" : 100, "count": 100,
"feed_url" : "http://www.JobServe.com/MySearch/C7B25D86D0844A.rss" "feed_url": "http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss"
}, },
{ {
/* "name" : "JobServe",*/
"count" : 100, "count": 100,
"feed_url" : "http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss" "feed_url": "http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss"
}, },
{ {
/* "name" : "JobServe",*/
"count" : 100, "count": 100,
"feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564698" "feed_url": "http://www.JobServe.com/MySearch/C7B25D86D0844A.rss"
}
,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564712"
}
,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21942123"
}
,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33166238"
}
,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888173"
}
,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.s1jobs.com/xml/m7dp711z2r.xml"
}
,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.s1jobs.com/xml/pfvf7o7z2r.xml"
}
,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.s1jobs.com/xml/lluqnt8z2r.xml"
}
,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.s1jobs.com/xml/tu33qt8z2r.xml"
}
,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.s1jobs.com/xml/u3btnz8z2r.xml"
}, },
{ {
/* "name" : "JobServe",*/
"count" : 100, "count": 100,
"feed_url" : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33256062" "feed_url": "http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss"
}, },
{ {
/* "name" : "JobServe",*/
"count" : 100, "count": 100,
"feed_url" : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33450169" "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564698"
}
,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34517029"
}, },
{ {
/* "name" : "JobServe",*/
"count" : 100, "count": 100,
"feed_url" : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888105" "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564712"
}
,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.technojobs.co.uk/rss.php/glasgow/searchtypeand/locationScotland/sortbyrelevant/jobtypeall"
}
,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss"
}
,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html,%20asp,%20web,%20sql,%20delphi,%20vb,%20vbscript,%20php,%20ajax,%20mysql,%20sqlserver,%20javascript,%20intranet,%20vmware,%20virtulization&location_include=Abu%20Dhabi&compare_resolved=RE_ABUDHABI_UNITEDARABEMIRATES&compare_search=Abu%20Dhabi&jobtype=X&search_emp_mkt_cd=ALL"
}
,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=London&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CO_LONDON&compare_search=London&search_emp_mkt_cd=ALL"
}
,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Glasgow&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=TO_G1_GLASGOW&compare_search=Glasgow&search_emp_mkt_cd=ALL"
}
,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&search_emp_mkt_cd=ALL"
} ,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Germany&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CY_GERMANY&compare_search=Germany&search_emp_mkt_cd=ALL"
}
,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml"
}
,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml"
}
,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.jobisjob.co.uk/rss?what=web+developer+javascript&where=glasgow&jobType=Contract"
}
,
{
/* "name" : "JobServe",*/
"count" : 100,
"feed_url" : "http://www.jobisjob.co.uk/rss?what=javascript&where=glasgow&jobType=Contract"
}, },
{ {
/* "name" : "JobServe",*/
"count" : 100, "count": 100,
"feed_url" : "http://www.jobisjob.co.uk/rss?what=asp&where=glasgow&jobType=Contract%2Ftemp" "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21942123"
}, },
{ {
/* "name" : "JobServe",*/
"count" : 100, "count": 100,
"feed_url" : "http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss" "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33166238"
}, },
{ {
/* "name" : "JobServe",*/
"count" : 100, "count": 100,
"feed_url" : "http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml" "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888173"
}, },
{ {
/* "name" : "JobServe",*/
"count" : 100, "count": 100,
"feed_url" : "http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml" "feed_url": "http://www.s1jobs.com/xml/m7dp711z2r.xml"
},
{
"count": 100,
"feed_url": "http://www.s1jobs.com/xml/pfvf7o7z2r.xml"
},
{
"count": 100,
"feed_url": "http://www.s1jobs.com/xml/lluqnt8z2r.xml"
},
{
"count": 100,
"feed_url": "http://www.s1jobs.com/xml/tu33qt8z2r.xml"
},
{
"count": 100,
"feed_url": "http://www.s1jobs.com/xml/u3btnz8z2r.xml"
},
{
"count": 100,
"feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33256062"
},
{
"count": 100,
"feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33450169"
},
{
"count": 100,
"feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34517029"
},
{
"count": 100,
"feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888105"
},
{
"count": 100,
"feed_url": "http://www.technojobs.co.uk/rss.php/glasgow/searchtypeand/locationScotland/sortbyrelevant/jobtypeall"
},
{
"count": 100,
"feed_url": "http://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss"
},
{
"count": 100,
"feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=London&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CO_LONDON&compare_search=London&search_emp_mkt_cd=ALL"
},
{
"count": 100,
"feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Glasgow&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=TO_G1_GLASGOW&compare_search=Glasgow&search_emp_mkt_cd=ALL"
},
{
"count": 100,
"feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&search_emp_mkt_cd=ALL"
},
{
"count": 100,
"feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Germany&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CY_GERMANY&compare_search=Germany&search_emp_mkt_cd=ALL"
},
{
"count": 100,
"feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml"
},
{
"count": 100,
"feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml"
},
{
"count": 100,
"feed_url": "http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss"
},
{
"count": 100,
"feed_url": "http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml"
},
{
"count": 100,
"feed_url": "http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml"
} }
]; ];
@ -192,9 +170,12 @@ let list = [
let newArray = list.map((item) => { let newArray = list.map((item) => {
let encoded = querystring.escape(item.feed_url); let u = url.parse(item.feed_url);
console.log(encoded); let newUrl = item.feed_url;
return {count:item.count, 'feed_url' : `http://52.211.105.9:6566/${encoded}`} if (u.host === 'www.jobsite.co.uk') {
newUrl = `http://52.211.105.9:6566/${querystring.escape(item.feed_url)}`;
}
return {count:item.count, 'feed_url' : newUrl}
}); });

317
jobs-corpus.js
View File

@ -1,7 +1,7 @@
var outputFile = 'jobs-special', RssBraider = require('rss-braider'), fs = require( var outputFile = 'jobs-special', RssBraider = require('rss-braider'), fs = require(
'fs'), ejs = require('ejs'), read = require('fs').readFileSync, join = require( 'fs'), ejs = require('ejs'), read = require('fs').readFileSync, join = require(
'path').join, str = read(join(__dirname, '/templates/rss.ejs'), 'path').join, str = read(join(__dirname, '/templates/rss.ejs'),
'utf8'), feeds = {}; 'utf8'), feeds = {};
var RSS = require('rss'); var RSS = require('rss');
var log4js = require('log4js'); var log4js = require('log4js');
var logger = log4js.getLogger(); var logger = log4js.getLogger();
@ -14,165 +14,165 @@ let corpus = {};
let sources = [ let sources = [
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.jobserve.com/MySearch/BAEBF3BDF82B8FEF.rss", 'feed_url': 'http://www.jobserve.com/MySearch/BAEBF3BDF82B8FEF.rss',
"page": "https://www.jobserve.com/gb/en/JobSearch.aspx?shid=A7ACEE7915E274717C" 'page': 'https://www.jobserve.com/gb/en/JobSearch.aspx?shid=A7ACEE7915E274717C'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=47820652" 'feed_url': 'http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=47820652'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.jobserve.com/MySearch/6DA9769BA89834AA.rss" 'feed_url': 'http://www.jobserve.com/MySearch/6DA9769BA89834AA.rss'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss" 'feed_url': 'http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss" 'feed_url': 'http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.JobServe.com/MySearch/C7B25D86D0844A.rss" 'feed_url': 'http://www.JobServe.com/MySearch/C7B25D86D0844A.rss'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss" 'feed_url': 'http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564698" 'feed_url': 'http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564698'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564712" 'feed_url': 'http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564712'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21942123" 'feed_url': 'http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21942123'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33166238" 'feed_url': 'http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33166238'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888173" 'feed_url': 'http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888173'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.s1jobs.com/xml/m7dp711z2r.xml" 'feed_url': 'http://www.s1jobs.com/xml/m7dp711z2r.xml'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.s1jobs.com/xml/pfvf7o7z2r.xml" 'feed_url': 'http://www.s1jobs.com/xml/pfvf7o7z2r.xml'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.s1jobs.com/xml/lluqnt8z2r.xml" 'feed_url': 'http://www.s1jobs.com/xml/lluqnt8z2r.xml'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.s1jobs.com/xml/tu33qt8z2r.xml" 'feed_url': 'http://www.s1jobs.com/xml/tu33qt8z2r.xml'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.s1jobs.com/xml/u3btnz8z2r.xml" 'feed_url': 'http://www.s1jobs.com/xml/u3btnz8z2r.xml'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33256062" 'feed_url': 'http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33256062'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33450169" 'feed_url': 'http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33450169'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34517029" 'feed_url': 'http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34517029'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888105" 'feed_url': 'http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888105'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.technojobs.co.uk/rss.php/glasgow/searchtypeand/locationScotland/sortbyrelevant/jobtypeall" 'feed_url': 'http://www.technojobs.co.uk/rss.php/glasgow/searchtypeand/locationScotland/sortbyrelevant/jobtypeall'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss" 'feed_url': 'http://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=London&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CO_LONDON&compare_search=London&search_emp_mkt_cd=ALL" 'feed_url': 'http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=London&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CO_LONDON&compare_search=London&search_emp_mkt_cd=ALL'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Glasgow&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=TO_G1_GLASGOW&compare_search=Glasgow&search_emp_mkt_cd=ALL" 'feed_url': 'http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Glasgow&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=TO_G1_GLASGOW&compare_search=Glasgow&search_emp_mkt_cd=ALL'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&search_emp_mkt_cd=ALL" 'feed_url': 'http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&search_emp_mkt_cd=ALL'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Germany&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CY_GERMANY&compare_search=Germany&search_emp_mkt_cd=ALL" 'feed_url': 'http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Germany&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CY_GERMANY&compare_search=Germany&search_emp_mkt_cd=ALL'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" 'feed_url': 'http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" 'feed_url': 'http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss" 'feed_url': 'http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml" 'feed_url': 'http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml" 'feed_url': 'http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml'
} }
]; ];
@ -181,104 +181,163 @@ sources = [
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.jobserve.com/MySearch/6DA9769BA89834AA.rss" 'feed_url': 'http://www.jobserve.com/MySearch/6DA9769BA89834AA.rss'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss" 'feed_url': 'http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss" 'feed_url': 'http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.JobServe.com/MySearch/C7B25D86D0844A.rss" 'feed_url': 'http://www.JobServe.com/MySearch/C7B25D86D0844A.rss'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss" 'feed_url': 'http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.s1jobs.com/xml/m7dp711z2r.xml" 'feed_url': 'http://www.s1jobs.com/xml/m7dp711z2r.xml'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.s1jobs.com/xml/pfvf7o7z2r.xml" 'feed_url': 'http://www.s1jobs.com/xml/pfvf7o7z2r.xml'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.s1jobs.com/xml/lluqnt8z2r.xml" 'feed_url': 'http://www.s1jobs.com/xml/lluqnt8z2r.xml'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.s1jobs.com/xml/tu33qt8z2r.xml" 'feed_url': 'http://www.s1jobs.com/xml/tu33qt8z2r.xml'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.s1jobs.com/xml/u3btnz8z2r.xml" 'feed_url': 'http://www.s1jobs.com/xml/u3btnz8z2r.xml'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" 'feed_url': 'http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml" 'feed_url': 'http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss" 'feed_url': 'http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml" 'feed_url': 'http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml'
}, },
{ {
"count": 100, 'count': 100,
"feed_url": "http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml" 'feed_url': 'http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml'
} }
]; ];
sources = [{'count': 100, 'feed_url': 'http://www.jobserve.com/MySearch/BAEBF3BDF82B8FEF.rss'}, {
'count': 100,
'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D47820652'
}, {'count': 100, 'feed_url': 'http://www.jobserve.com/MySearch/6DA9769BA89834AA.rss'}, {
'count': 100,
'feed_url': 'http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss'
}, {'count': 100, 'feed_url': 'http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss'}, {
'count': 100,
'feed_url': 'http://www.JobServe.com/MySearch/C7B25D86D0844A.rss'
}, {'count': 100, 'feed_url': 'http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss'}, {
'count': 100,
'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D21564698'
}, {
'count': 100,
'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D21564712'
}, {
'count': 100,
'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D21942123'
}, {
'count': 100,
'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D33166238'
}, {
'count': 100,
'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D34888173'
}, {'count': 100, 'feed_url': 'http://www.s1jobs.com/xml/m7dp711z2r.xml'}, {
'count': 100,
'feed_url': 'http://www.s1jobs.com/xml/pfvf7o7z2r.xml'
}, {'count': 100, 'feed_url': 'http://www.s1jobs.com/xml/lluqnt8z2r.xml'}, {
'count': 100,
'feed_url': 'http://www.s1jobs.com/xml/tu33qt8z2r.xml'
}, {'count': 100, 'feed_url': 'http://www.s1jobs.com/xml/u3btnz8z2r.xml'}, {
'count': 100,
'feed_url': 'http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33256062'
}, {'count': 100, 'feed_url': 'http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33450169'}, {
'count': 100,
'feed_url': 'http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34517029'
}, {'count': 100, 'feed_url': 'http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888105'}, {
'count': 100,
'feed_url': 'http://www.technojobs.co.uk/rss.php/glasgow/searchtypeand/locationScotland/sortbyrelevant/jobtypeall'
}, {'count': 100, 'feed_url': 'http://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss'}, {
'count': 100,
'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_include%3DLondon%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26compare_resolved%3DCO_LONDON%26compare_search%3DLondon%26search_emp_mkt_cd%3DALL'
}, {
'count': 100,
'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_include%3DGlasgow%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26compare_resolved%3DTO_G1_GLASGOW%26compare_search%3DGlasgow%26search_emp_mkt_cd%3DALL'
}, {
'count': 100,
'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26search_emp_mkt_cd%3DALL'
}, {
'count': 100,
'feed_url': 'http://52.211.105.9:6566/http%3A%2F%2Fwww.jobsite.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26skill_atleast%3Dhtml%2520%2C%2520asp%2520%2C%2520web%2520%2C%2520sql%2520%2C%2520delphi%2520%2C%2520vb%2520%2C%2520vbscript%2520%2C%2520php%2520%2C%2520ajax%2520%2C%2520mysql%2520%2C%2520sqlserver%2520%2C%2520javascript%2520%2C%2520intranet%2520%2C%2520vmware%2520%2C%2520virtulization%26location_include%3DGermany%26location_within%3D10%26reqd_salary%3DANY%7C%26daysback%3D7%26scc%3DUK%26compare_resolved%3DCY_GERMANY%26compare_search%3DGermany%26search_emp_mkt_cd%3DALL'
}, {'count': 100, 'feed_url': 'http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml'}, {
'count': 100,
'feed_url': 'http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml'
}, {'count': 100, 'feed_url': 'http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss'}, {
'count': 100,
'feed_url': 'http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml'
}, {'count': 100, 'feed_url': 'http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml'}];
// Pull feeds from config files: // Pull feeds from config files:
// feeds.simple_test_feed = require("./config/feed").feed; // feeds.simple_test_feed = require("./config/feed").feed;
// Or define in-line // Or define in-line
feeds.simple_test_feed = { feeds.simple_test_feed = {
"feed_name": "feed", "default_count": 1, "no_cdata_fields": [], // Don't wrap these fields in CDATA tags 'feed_name': 'feed', 'default_count': 1, 'no_cdata_fields': [], // Don't wrap these fields in CDATA tags
"plugins": [ 'plugins': [
'filter_location', 'filter_reject', 'filter_md_jobs', 'filter_today_only' 'filter_location', 'filter_reject', 'filter_md_jobs', 'filter_today_only'
/*'filter_location', 'filter_today_only'*/ /*'filter_location', 'filter_today_only'*/
], "meta": { ], 'meta': {
"title": "Jobs", 'title': 'Jobs',
"description": "Combined Jobs Feed", 'description': 'Combined Jobs Feed',
'site_url': 'http://pipes.silvrtree.co.uk/jobs-special.xml' 'site_url': 'http://pipes.silvrtree.co.uk/jobs-special.xml'
}, "sources": sources }, 'sources': sources
}; };
var braider_options = { var braider_options = {
feeds: feeds, feeds: feeds,
indent: " ", indent: ' ',
date_sort_order: "desc", // Newest first date_sort_order: 'desc', // Newest first
log_level: "debug", log_level: 'debug',
dedupe_fields: ['link', 'guid'], dedupe_fields: ['link', 'guid'],
plugins_directories: [__dirname + "/plugins/"] plugins_directories: [__dirname + '/plugins/']
}; };
console.log('Working..'); console.log('Working..');
var rss_braider = RssBraider.createClient(braider_options); var rss_braider = RssBraider.createClient(braider_options);
@ -302,7 +361,7 @@ function doFeeds() {
}*/ }*/
let ejsOutput = ejs.compile(str)(j); let ejsOutput = ejs.compile(str)(j);
fs.writeFile(__dirname + "/dist/" + outputFile + ".html", fs.writeFile(__dirname + '/dist/' + outputFile + '.html',
ejsOutput, ejsOutput,
function (err) { function (err) {
@ -310,10 +369,10 @@ function doFeeds() {
return console.log(err); return console.log(err);
} }
console.log("The file was saved!"); console.log('The file was saved!');
}); });
fs.writeFile(__dirname + "/dist/" + outputFile + ".json", fs.writeFile(__dirname + '/dist/' + outputFile + '.json',
data, data,
function (err) { function (err) {
if (err) { if (err) {
@ -321,21 +380,21 @@ function doFeeds() {
return console.log(err); return console.log(err);
} }
console.log("The file was saved!"); console.log('The file was saved!');
}); });
saveCorpus(); saveCorpus();
let newfeed = new RSS({title:'Jobs Corpus'}, j.items); let newfeed = new RSS({title: 'Jobs Corpus'}, j.items);
// console.log(newfeed); // console.log(newfeed);
fs.writeFile(__dirname + "/dist/" + outputFile + ".xml", newfeed.xml(' '), function(err) { fs.writeFile(__dirname + '/dist/' + outputFile + '.xml', newfeed.xml(' '), function (err) {
if (err) { if (err) {
return console.log(err); return console.log(err);
} }
console.log("The XML file was saved!"); console.log('The XML file was saved!');
}); });
@ -344,15 +403,14 @@ function doFeeds() {
} }
function scoreEntry(s) { function scoreEntry(s) {
let score = 0; let score = 0;
let used = []; let used = [];
for(let t of s) { for (let t of s) {
//console.log(`processing: ${t}`); //console.log(`processing: ${t}`);
if (!corpus.hasOwnProperty(t)) { if (!corpus.hasOwnProperty(t)) {
corpus[t] = {score:0, common:1}; corpus[t] = {score: 0, common: 1};
} else { } else {
corpus[t].common++; corpus[t].common++;
if (used.indexOf(t) === -1) { if (used.indexOf(t) === -1) {
@ -365,6 +423,7 @@ function scoreEntry(s) {
return score; return score;
} }
function processJson(j) { function processJson(j) {
// console.log(j); // console.log(j);
console.log('Processing...'); console.log('Processing...');
@ -383,6 +442,7 @@ function processJson(j) {
return j; return j;
} }
function getNoneStopWords(sentence) { function getNoneStopWords(sentence) {
let common = getStopWords(); let common = getStopWords();
let wordArr = sentence.match(/\w+/g), let wordArr = sentence.match(/\w+/g),
@ -391,7 +451,7 @@ function getNoneStopWords(sentence) {
word, i; word, i;
for (i = 0; i < common.length; i++) { for (i = 0; i < common.length; i++) {
commonObj[ common[i].trim() ] = true; commonObj[common[i].trim()] = true;
} }
for (i = 0; i < wordArr.length; i++) { for (i = 0; i < wordArr.length; i++) {
@ -408,24 +468,21 @@ function getStopWords() {
} }
function doFeedsXML() { function doFeedsXML() {
rss_braider.processFeed('simple_test_feed', 'rss', function(err, data) { rss_braider.processFeed('simple_test_feed', 'rss', function (err, data) {
if (err) { if (err) {
return console.log(err); return console.log(err);
} }
console.log('Saving', __dirname + '/dist/' + outputFile + '.xml');
console.log('Saving', __dirname + "/dist/" + outputFile + ".xml"); fs.writeFile(__dirname + '/dist/' + outputFile + '.xml', data, function (err) {
fs.writeFile(__dirname + "/dist/" + outputFile + ".xml", data, function(err) {
if (err) { if (err) {
return console.log(err); return console.log(err);
} }
console.log("The file was saved!"); console.log('The file was saved!');
}); });
}); });
@ -433,10 +490,10 @@ function doFeedsXML() {
function loadCorpus() { function loadCorpus() {
let fileName = __dirname + "/dist/corpus.json"; let fileName = __dirname + '/dist/corpus.json';
console.log(`Loading ${fileName}`); console.log(`Loading ${fileName}`);
jsonfile.readFile(fileName, function(err, obj) { jsonfile.readFile(fileName, function (err, obj) {
corpus = obj; corpus = obj;
let entries = Object.keys(corpus).length; let entries = Object.keys(corpus).length;
console.log(`Corpus loaded (${entries} entries)`); console.log(`Corpus loaded (${entries} entries)`);
@ -445,9 +502,9 @@ function loadCorpus() {
} }
function saveCorpus() { function saveCorpus() {
let fileName = __dirname + "/dist/corpus.json"; let fileName = __dirname + '/dist/corpus.json';
console.log(`Saving ${fileName}`); console.log(`Saving ${fileName}`);
jsonfile.writeFile(fileName, corpus,function(err, obj) { jsonfile.writeFile(fileName, corpus, function (err, obj) {
console.error(err); console.error(err);
console.log(obj); console.log(obj);
}); });