martin/jobscraper
martin/jobscraper
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added scrapers for CWJobs ( totaljobs reskin ) and technojobs

Browse Source
This commit is contained in:
Martin Donnelly 2020-05-25 00:36:38 +01:00
parent 460b7b9ed2
commit fdb291838b
4 changed files with 109 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
grabber.js
View File

@ -8,25 +8,33 @@
const CronJob = require('cron').CronJob;
const IndeedScraper = require('./scrapers/indeed');
const TotaljobsScraper = require('./scrapers/totaljobs');
const CwjobsScraper = require('./scrapers/cwjobs');
const JobserveScraper = require('./scrapers/rss.jobserve');
const RssS1Jobs = require('./scrapers/rss.s1jobs');
const RssTechnojobs = require('./scrapers/rss.technojobs');
(async function () {
console.log('Started..');
const indeedScraper = new IndeedScraper();
const totaljobsScraper = new TotaljobsScraper();
const cwjobsScraper = new CwjobsScraper();
const jobserveScraper = new JobserveScraper();
const s1jobsScraper = new RssS1Jobs();
const technojobsScraper = new RssTechnojobs();
new CronJob('5 6-23/3 * * *', async function() {
await indeedScraper.go('london');
await totaljobsScraper.go('london');
await cwjobsScraper.go('london');
await indeedScraper.go('glasgow');
await totaljobsScraper.go('glasgow');
await cwjobsScraper.go('glasgow');
await indeedScraper.go('edinburgh');
await totaljobsScraper.go('edinburgh');
await cwjobsScraper.go('edinburgh');
await indeedScraper.go('milton keynes');
await totaljobsScraper.go('milton keynes');
await cwjobsScraper.go('milton keynes');
}, null, true);
new CronJob('0 6-23/1 * * *', async function() {
@ -50,5 +58,9 @@ const RssS1Jobs = require('./scrapers/rss.s1jobs');
await s1jobsScraper.go('http://www.s1jobs.com/xml/u3btnz8z2r.xml');
await s1jobsScraper.go('http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml');
await s1jobsScraper.go('http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml');
await technojobsScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationglasgow/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1')
await technojobsScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationLONDON/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1')
await technojobsScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationMilton%20Keynes/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1')
}, null, true);
})();

27
test/cwjobs.js Normal file
View File

@ -0,0 +1,27 @@
/**
* Created by WebStorm.
* User: martin
* Date: 24/05/2020
* Time: 23:46
*/
const tape = require('tape');
const _test = require('tape-promise').default; // <---- notice 'default'
const test = _test(tape); // decorate tape
const fs = require('fs');
const cheerio = require('cheerio');
const CwjobsScraper = require('../scrapers/cwjobs');
const cwjobsScraper = new CwjobsScraper();
// const page = fs.readFileSync('data/indeed/indeed-2020-04-16--092311.html');
console.log(`${__dirname}`);
// const page = fs.readFileSync(`${__dirname}/data/totaljobs/totaljobs-2020-04-16--121504.html`);
test.test('Test CWJobs scraper', async t => {
await cwjobsScraper.go('london');
t.end();
});

35
test/data/technojobs/page1 Normal file
View File

@ -0,0 +1,35 @@
<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0"><channel><title>html OR node OR web OR sql OR delphi OR javascript OR ajax : Search : Technojobs.co.uk</title><link>https://www.technojobs.co.uk/search.phtml/html+OR+node+OR+web+OR+sql+OR+delphi+OR+javascript+OR+ajax/excludekeywords/locationglasgow/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1</link><language>en-GB</language><description>Live job search results for 'html OR node OR web OR sql OR delphi OR javascript OR ajax' performed by Technojobs.co.uk</description><lastBuildDate>Thu, 14 May 2020 08:40:00 GMT</lastBuildDate>
<item><title>.Net Developer - Edinburgh</title><link><![CDATA[https://www.technojobs.co.uk/job_mobile.phtml/2825559]]></link><description><![CDATA[
<p>Salary/Rate: &pound;35,000 - &pound;65,000</p><p>Location: Glasgow</p><p>Edinburgh Based Role Mid-Level and Senior Developer required Salaries &pound;35-65k My client specialises in a Financial software platform designed with the intention to reduce complexity and increase the automation in turn...</p><br />
]]></description><pubDate>Thu, 14 May 2020 08:40:00 GMT</pubDate></item>
<item><title>Dynamics AX Support Consultant &amp;amp;pound;40,000 - Glasgow </title><link><![CDATA[https://www.technojobs.co.uk/job_mobile.phtml/2825170]]></link><description><![CDATA[
<p>Salary/Rate: </p><p>Location: Glasgow</p><p>Dynamics AX Support Consultant &pound;40,000 - Glasgow Active requirement for an AX Support Developer to join the team at one of the UKs leading MS Gold Partner's. Based from their growing Glasgow offices, the role offers a great...</p><br />
]]></description><pubDate>Tue, 12 May 2020 14:52:00 GMT</pubDate></item>
<item><title>Software Engineer</title><link><![CDATA[https://www.technojobs.co.uk/job_mobile.phtml/2824080]]></link><description><![CDATA[
<p>Salary/Rate: &pound;30,000 - &pound;40,000</p><p>Location: Glasgow</p><p>My client is a global engineering company bringing over a century of experience within their cutting edge developments through their digital solutions. Their products help multiple sectors improve their everyday processes. You...</p><br />
]]></description><pubDate>Thu, 07 May 2020 09:20:00 GMT</pubDate></item>
<item><title>Software Developer</title><link><![CDATA[https://www.technojobs.co.uk/job_mobile.phtml/2823454]]></link><description><![CDATA[
<p>Salary/Rate: &pound;30,000 - &pound;45,000</p><p>Location: Motherwell</p><p>My market leading client first broke into the market back in 2011 and have since grown from strength to strength providing a one of a kind service to the retail market. After a recent global expansion they are looking to grow their Glasgow...</p><br />
]]></description><pubDate>Mon, 04 May 2020 16:58:00 GMT</pubDate></item>
<item><title>.Net/C# Developer Glasgow</title><link><![CDATA[https://www.technojobs.co.uk/job_mobile.phtml/2823453]]></link><description><![CDATA[
<p>Salary/Rate: &pound;35,000 - &pound;50,000</p><p>Location: Glasgow</p><p>My market leading client first broke into the market back in 2011 and have since grown from strength to strength providing a one of a kind service to the retail market. After a recent global expansion they are looking to grow their Glasgow...</p><br />
]]></description><pubDate>Mon, 04 May 2020 16:58:00 GMT</pubDate></item>
<item><title>Dynamics NAV Consultant - Home Based - up to &amp;amp;pound;70,000 </title><link><![CDATA[https://www.technojobs.co.uk/job_mobile.phtml/2823263]]></link><description><![CDATA[
<p>Salary/Rate: </p><p>Location: Glasgow</p><p>Dynamics NAV Consultant - Home Based - up to &pound;70,000 I am currently recruiting for an enthusiastic Dynamics NAV Consultant to join my client who specialise in providing the most progressive solutions for Microsoft Dynamics...</p><br />
]]></description><pubDate>Mon, 04 May 2020 10:33:00 GMT</pubDate></item>
<item><title>DevOps Engineer - Glasgow and Edinburgh</title><link><![CDATA[https://www.technojobs.co.uk/job_mobile.phtml/2822423]]></link><description><![CDATA[
<p>Salary/Rate: &pound;25,000 - &pound;50,000</p><p>Location: Glasgow</p><p>I am currently working with a client in Glasgow looking to hire AWS Developers / Consultants to join their expanding team. They are a rapidly growing consultancy based in Glasgow with clients in Edinburgh and across the central belt....</p><br />
]]></description><pubDate>Wed, 29 Apr 2020 08:00:00 GMT</pubDate></item>
<item><title>Dynamics AX (365) Functional Consultant - &amp;amp;pound;45k-&amp;amp;pound;75k</title><link><![CDATA[https://www.technojobs.co.uk/job_mobile.phtml/2821908]]></link><description><![CDATA[
<p>Salary/Rate: </p><p>Location: Glasgow</p><p>Dynamics AX (365) Functional Consultant - &pound;45k-&pound;75k An internationally renowned MS Gold Partner are currently resourcing for an AX Functional Consultants to deliver multiple AX 365 projects. The successful...</p><br />
]]></description><pubDate>Mon, 27 Apr 2020 11:19:00 GMT</pubDate></item>
<item><title>Dynamics NAV Developer - End User </title><link><![CDATA[https://www.technojobs.co.uk/job_mobile.phtml/2821899]]></link><description><![CDATA[
<p>Salary/Rate: &pound;40,000 - &pound;60,000</p><p>Location: Paisley</p><p>Dynamics NAV Developer, Paisley, &pound;45-60K + Benefits Ahead of a multi-site Dynamics NAV upgrade project scheduled to begin in the coming months, a nationally-recognised business are now recruiting for an ambitious Dynamics...</p><br />
]]></description><pubDate>Mon, 27 Apr 2020 11:11:00 GMT</pubDate></item>
<item><title>PHP Developer Glasgow</title><link><![CDATA[https://www.technojobs.co.uk/job_mobile.phtml/2821695]]></link><description><![CDATA[
<p>Salary/Rate: &pound;40,000 - &pound;45,000</p><p>Location: Glasgow</p><p>PHP Developer - Glasgow My Client is looking for an enthusiastic and talented PHP Developer to join their growing development team. You'll be working with (and learning from!) a skilled and experienced team on some exciting projects,...</p><br />
]]></description><pubDate>Sun, 26 Apr 2020 16:06:00 GMT</pubDate></item>
<item><title>PHP Developer Glasgow</title><link><![CDATA[https://www.technojobs.co.uk/job_mobile.phtml/2821694]]></link><description><![CDATA[
<p>Salary/Rate: &pound;40,000 - &pound;45,000</p><p>Location: Glasgow</p><p>PHP Developer - Glasgow My Client is looking for an enthusiastic and talented PHP Developer to join their growing development team. You'll be working with (and learning from!) a skilled and experienced team on some exciting projects,...</p><br />
]]></description><pubDate>Sun, 26 Apr 2020 16:04:00 GMT</pubDate></item></channel></rss>

35
test/technojobs.js Normal file
View File

@ -0,0 +1,35 @@
/**
* Created by WebStorm.
* User: martin
* Date: 24/05/2020
* Time: 23:12
*/
const tape = require('tape');
const _test = require('tape-promise').default; // <---- notice 'default'
const test = _test(tape); // decorate tape
const fs = require('fs');
const cheerio = require('cheerio');
const RssTechnojobs = require('../scrapers/rss.technojobs');
const testScraper = new RssTechnojobs();
const feed = fs.readFileSync('test/data/technojobs/page1');
test.test('Test Technojobs scraper', async t => {
// await testScraper.loadFeed(feed);
// testScraper.reduceItems();
// await s1jobsScraper.filterAdverts();
// await s1jobsScraper.addToDB();
await testScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationglasgow/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1')
await testScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationLONDON/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1')
await testScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationMilton%20Keynes/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1')
t.end();
});