From fdb291838bb254fc9fb7f3e6ddc290f1b9082639 Mon Sep 17 00:00:00 2001 From: Martin Donnelly Date: Mon, 25 May 2020 00:36:38 +0100 Subject: [PATCH] Added scrapers for CWJobs ( totaljobs reskin ) and technojobs --- grabber.js | 12 ++++++++++++ test/cwjobs.js | 27 +++++++++++++++++++++++++++ test/data/technojobs/page1 | 35 +++++++++++++++++++++++++++++++++++ test/technojobs.js | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 109 insertions(+) create mode 100644 test/cwjobs.js create mode 100644 test/data/technojobs/page1 create mode 100644 test/technojobs.js diff --git a/grabber.js b/grabber.js index 9381984..ba5ce79 100644 --- a/grabber.js +++ b/grabber.js @@ -8,25 +8,33 @@ const CronJob = require('cron').CronJob; const IndeedScraper = require('./scrapers/indeed'); const TotaljobsScraper = require('./scrapers/totaljobs'); +const CwjobsScraper = require('./scrapers/cwjobs'); const JobserveScraper = require('./scrapers/rss.jobserve'); const RssS1Jobs = require('./scrapers/rss.s1jobs'); +const RssTechnojobs = require('./scrapers/rss.technojobs'); (async function () { console.log('Started..'); const indeedScraper = new IndeedScraper(); const totaljobsScraper = new TotaljobsScraper(); + const cwjobsScraper = new CwjobsScraper(); const jobserveScraper = new JobserveScraper(); const s1jobsScraper = new RssS1Jobs(); + const technojobsScraper = new RssTechnojobs(); new CronJob('5 6-23/3 * * *', async function() { await indeedScraper.go('london'); await totaljobsScraper.go('london'); + await cwjobsScraper.go('london'); await indeedScraper.go('glasgow'); await totaljobsScraper.go('glasgow'); + await cwjobsScraper.go('glasgow'); await indeedScraper.go('edinburgh'); await totaljobsScraper.go('edinburgh'); + await cwjobsScraper.go('edinburgh'); await indeedScraper.go('milton keynes'); await totaljobsScraper.go('milton keynes'); + await cwjobsScraper.go('milton keynes'); }, null, true); new CronJob('0 6-23/1 * * *', async function() { @@ -50,5 +58,9 @@ const RssS1Jobs = require('./scrapers/rss.s1jobs'); await s1jobsScraper.go('http://www.s1jobs.com/xml/u3btnz8z2r.xml'); await s1jobsScraper.go('http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml'); await s1jobsScraper.go('http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml'); + + await technojobsScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationglasgow/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1') + await technojobsScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationLONDON/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1') + await technojobsScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationMilton%20Keynes/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1') }, null, true); })(); diff --git a/test/cwjobs.js b/test/cwjobs.js new file mode 100644 index 0000000..ec74e23 --- /dev/null +++ b/test/cwjobs.js @@ -0,0 +1,27 @@ +/** + * Created by WebStorm. + * User: martin + * Date: 24/05/2020 + * Time: 23:46 + + */ +const tape = require('tape'); +const _test = require('tape-promise').default; // <---- notice 'default' +const test = _test(tape); // decorate tape + +const fs = require('fs'); +const cheerio = require('cheerio'); + +const CwjobsScraper = require('../scrapers/cwjobs'); + +const cwjobsScraper = new CwjobsScraper(); + +// const page = fs.readFileSync('data/indeed/indeed-2020-04-16--092311.html'); +console.log(`${__dirname}`); +// const page = fs.readFileSync(`${__dirname}/data/totaljobs/totaljobs-2020-04-16--121504.html`); + +test.test('Test CWJobs scraper', async t => { + await cwjobsScraper.go('london'); + + t.end(); +}); diff --git a/test/data/technojobs/page1 b/test/data/technojobs/page1 new file mode 100644 index 0000000..a2b5ef0 --- /dev/null +++ b/test/data/technojobs/page1 @@ -0,0 +1,35 @@ + + html OR node OR web OR sql OR delphi OR javascript OR ajax : Search : Technojobs.co.ukhttps://www.technojobs.co.uk/search.phtml/html+OR+node+OR+web+OR+sql+OR+delphi+OR+javascript+OR+ajax/excludekeywords/locationglasgow/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1en-GBLive job search results for 'html OR node OR web OR sql OR delphi OR javascript OR ajax' performed by Technojobs.co.ukThu, 14 May 2020 08:40:00 GMT +.Net Developer - EdinburghSalary/Rate: £35,000 - £65,000

Location: Glasgow

Edinburgh Based Role Mid-Level and Senior Developer required Salaries £35-65k My client specialises in a Financial software platform designed with the intention to reduce complexity and increase the automation in turn...


+]]>Thu, 14 May 2020 08:40:00 GMT +Dynamics AX Support Consultant &amp;pound;40,000 - Glasgow Salary/Rate:

Location: Glasgow

Dynamics AX Support Consultant £40,000 - Glasgow Active requirement for an AX Support Developer to join the team at one of the UKs leading MS Gold Partner's. Based from their growing Glasgow offices, the role offers a great...


+]]>Tue, 12 May 2020 14:52:00 GMT +Software EngineerSalary/Rate: £30,000 - £40,000

Location: Glasgow

My client is a global engineering company bringing over a century of experience within their cutting edge developments through their digital solutions. Their products help multiple sectors improve their everyday processes. You...


+]]>Thu, 07 May 2020 09:20:00 GMT +Software DeveloperSalary/Rate: £30,000 - £45,000

Location: Motherwell

My market leading client first broke into the market back in 2011 and have since grown from strength to strength providing a one of a kind service to the retail market. After a recent global expansion they are looking to grow their Glasgow...


+]]>Mon, 04 May 2020 16:58:00 GMT +.Net/C# Developer GlasgowSalary/Rate: £35,000 - £50,000

Location: Glasgow

My market leading client first broke into the market back in 2011 and have since grown from strength to strength providing a one of a kind service to the retail market. After a recent global expansion they are looking to grow their Glasgow...


+]]>Mon, 04 May 2020 16:58:00 GMT +Dynamics NAV Consultant - Home Based - up to &amp;pound;70,000 Salary/Rate:

Location: Glasgow

Dynamics NAV Consultant - Home Based - up to £70,000 I am currently recruiting for an enthusiastic Dynamics NAV Consultant to join my client who specialise in providing the most progressive solutions for Microsoft Dynamics...


+]]>Mon, 04 May 2020 10:33:00 GMT +DevOps Engineer - Glasgow and EdinburghSalary/Rate: £25,000 - £50,000

Location: Glasgow

I am currently working with a client in Glasgow looking to hire AWS Developers / Consultants to join their expanding team. They are a rapidly growing consultancy based in Glasgow with clients in Edinburgh and across the central belt....


+]]>Wed, 29 Apr 2020 08:00:00 GMT +Dynamics AX (365) Functional Consultant - &amp;pound;45k-&amp;pound;75kSalary/Rate:

Location: Glasgow

Dynamics AX (365) Functional Consultant - £45k-£75k An internationally renowned MS Gold Partner are currently resourcing for an AX Functional Consultants to deliver multiple AX 365 projects. The successful...


+]]>Mon, 27 Apr 2020 11:19:00 GMT +Dynamics NAV Developer - End User Salary/Rate: £40,000 - £60,000

Location: Paisley

Dynamics NAV Developer, Paisley, £45-60K + Benefits Ahead of a multi-site Dynamics NAV upgrade project scheduled to begin in the coming months, a nationally-recognised business are now recruiting for an ambitious Dynamics...


+]]>Mon, 27 Apr 2020 11:11:00 GMT +PHP Developer GlasgowSalary/Rate: £40,000 - £45,000

Location: Glasgow

PHP Developer - Glasgow My Client is looking for an enthusiastic and talented PHP Developer to join their growing development team. You'll be working with (and learning from!) a skilled and experienced team on some exciting projects,...


+]]>Sun, 26 Apr 2020 16:06:00 GMT +PHP Developer GlasgowSalary/Rate: £40,000 - £45,000

Location: Glasgow

PHP Developer - Glasgow My Client is looking for an enthusiastic and talented PHP Developer to join their growing development team. You'll be working with (and learning from!) a skilled and experienced team on some exciting projects,...


+]]>Sun, 26 Apr 2020 16:04:00 GMT \ No newline at end of file diff --git a/test/technojobs.js b/test/technojobs.js new file mode 100644 index 0000000..29e2732 --- /dev/null +++ b/test/technojobs.js @@ -0,0 +1,35 @@ +/** + * Created by WebStorm. + * User: martin + * Date: 24/05/2020 + * Time: 23:12 + + */ +const tape = require('tape'); +const _test = require('tape-promise').default; // <---- notice 'default' +const test = _test(tape); // decorate tape + +const fs = require('fs'); +const cheerio = require('cheerio'); + +const RssTechnojobs = require('../scrapers/rss.technojobs'); + +const testScraper = new RssTechnojobs(); + +const feed = fs.readFileSync('test/data/technojobs/page1'); + +test.test('Test Technojobs scraper', async t => { + // await testScraper.loadFeed(feed); + + // testScraper.reduceItems(); + + // await s1jobsScraper.filterAdverts(); + // await s1jobsScraper.addToDB(); + + await testScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationglasgow/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1') + await testScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationLONDON/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1') + await testScraper.go('https://www.technojobs.co.uk/rss.php/html%20OR%20node%20OR%20web%20OR%20sql%20OR%20delphi%20OR%20javascript%20OR%20ajax/excludekeywords/locationMilton%20Keynes/radius25/termsin0/salary0/postedwithinall/jobtypeall/searchfieldRSearchIndex/page1') + + + t.end(); +});