From 17a348f625cf0db54e3085d80f83ea3d2c498d54 Mon Sep 17 00:00:00 2001 From: Martin Donnelly Date: Thu, 10 Dec 2020 16:05:21 +0000 Subject: [PATCH] Added angular to indeed, cws and total Added a new search for jobserve --- grabber.js | 1 + scrapers/cwjobs.js | 2 +- scrapers/indeed.js | 24 ++++++++++++------------ scrapers/totaljobs.js | 2 +- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/grabber.js b/grabber.js index 3637428..1746d91 100644 --- a/grabber.js +++ b/grabber.js @@ -38,6 +38,7 @@ const RssTechnojobs = require('./scrapers/rss.technojobs'); }, null, true); new CronJob('0 6-23/1 * * *', async function() { + await jobserveScraper.go('https://www.jobserve.com/MySearch/D48462060FB24B6C.rss'); await jobserveScraper.go('https://www.jobserve.com/MySearch/BAEBF3BDF82B8FEF.rss'); await jobserveScraper.go('https://www.jobserve.com/MySearch/9BCBF25C586A0E3F.rss'); await jobserveScraper.go('https://www.jobserve.com/MySearch/F3A56475D5FD4966.rss'); diff --git a/scrapers/cwjobs.js b/scrapers/cwjobs.js index 71db3e2..c80b9db 100644 --- a/scrapers/cwjobs.js +++ b/scrapers/cwjobs.js @@ -20,7 +20,7 @@ class CwjobsScraper extends TotaljobsScraper { } async go(location = 'london') { - this.setStartUrl(`https://www.cwjobs.co.uk/jobs/contract/html-or-vue-or-vuejs-or-web-or-sql-or-delphi-or-vb-or-vbscript-or-php-or-ajax-or-mysql-or-sqlserver-or-javascript-or-node-or-nodejs-or-svelte-or-sveltejs-not-react/in-${encodeURIComponent(location)}?q=Html+Or+Vue+Or+Vuejs+Or+Web+Or+Sql+Or+Delphi+Or+Vb+Or+Vbscript+Or+Php+Or+Ajax+Or+Mysql+Or+Sqlserver+Or+Javascript+Or+Node+Or+Nodejs+Or+Svelte+Or+Sveltejs+NOT+React&postedwithin=3&radius=20`); + this.setStartUrl(`https://www.cwjobs.co.uk/jobs/contract/html-or-angular-or-vue-or-vuejs-or-web-or-sql-or-delphi-or-vb-or-vbscript-or-php-or-ajax-or-mysql-or-sqlserver-or-javascript-or-node-or-nodejs-or-svelte-or-sveltejs-not-react/in-${encodeURIComponent(location)}?q=Html+Or+Vue+Or+Vuejs+Or+Web+Or+Sql+Or+Delphi+Or+Vb+Or+Vbscript+Or+Php+Or+Ajax+Or+Mysql+Or+Sqlserver+Or+Javascript+Or+Node+Or+Nodejs+Or+Svelte+Or+Sveltejs+NOT+React&postedwithin=3&radius=20`); // this.setStartUrl('https://www.indeed.co.uk/jobs?as_and=&as_phr=&as_any=javascript+nodejs&as_not=&as_ttl=&as_cmp=&jt=contract&st=&as_src=&salary=&radius=25&l=london&fromage=7&limit=10&sort=date&psf=advsrch&from=advancedsearch'); // Glasgow diff --git a/scrapers/indeed.js b/scrapers/indeed.js index a9f3e83..2d236e2 100644 --- a/scrapers/indeed.js +++ b/scrapers/indeed.js @@ -11,7 +11,7 @@ const cheerio = require('cheerio'); const MasterScraper = require('../lib/scraper'); class IndeedScraper extends MasterScraper { - + constructor() { super(); this.siteurl = 'www.indeed.co.uk'; @@ -23,15 +23,15 @@ class IndeedScraper extends MasterScraper { this.antiAd = /sja\d+/gi; } - + // Site specific parts below here - + async breakPage() { const $ = this.currentPage; const ads = []; - + const sections = $('div.row.result'); - + await sections.each(async (index, item) => { // console.log($(item).html()); const ad = await this.extractDetails(item); @@ -73,7 +73,7 @@ class IndeedScraper extends MasterScraper { return newObj; } - + async getIndividualPage(item) { const newItem = {...item}; console.log('Getting', item.url); @@ -100,10 +100,10 @@ class IndeedScraper extends MasterScraper { async checkNext() { const $ = this.currentPage; const next = $('.pagination > *:last-child').attr('href') || ''; - if (next !== '') + if (next !== '') // next = `https://${ this.siteurl }${next}`; this.makeUrl(next); - + console.log(next); } @@ -122,9 +122,9 @@ class IndeedScraper extends MasterScraper { await this.checkNext(); await this.getJobPages(); - + // nextPage = await this.checkNext(); - + // if (nextPage === previousPage) nextPage = ''; // this.setStartUrl(nextPage); @@ -137,7 +137,7 @@ class IndeedScraper extends MasterScraper { } async go(location = 'london') { - this.setStartUrl(`https://www.indeed.co.uk/jobs?as_and=&as_phr=&as_any=Html+Web+Sql+Delphi+Vb+Vbscript+Php+Ajax+Mysql+Sqlserver+Javascript+Nodejs+vuejs+sveltejs&as_not=React&as_ttl=&as_cmp=&jt=contract&st=&as_src=&salary=&radius=0&l=${encodeURIComponent(location)}&fromage=1&limit=50&sort=&psf=advsrch&from=advancedsearch`); + this.setStartUrl(`https://www.indeed.co.uk/jobs?as_and=&as_phr=&as_any=Angular+Html+Web+Sql+Delphi+Vb+Vbscript+Php+Ajax+Mysql+Sqlserver+Javascript+Nodejs+vuejs+sveltejs&as_not=React&as_ttl=&as_cmp=&jt=contract&st=&as_src=&salary=&radius=0&l=${encodeURIComponent(location)}&fromage=1&limit=50&sort=&psf=advsrch&from=advancedsearch`); await this.processSite().catch((err) => { console.error('Indeed Go', err); @@ -145,7 +145,7 @@ class IndeedScraper extends MasterScraper { console.log(`Indeed ${location} completed`); } - + } module.exports = IndeedScraper; diff --git a/scrapers/totaljobs.js b/scrapers/totaljobs.js index ff12f2e..67db3d1 100644 --- a/scrapers/totaljobs.js +++ b/scrapers/totaljobs.js @@ -155,7 +155,7 @@ class TotaljobsScraper extends MasterScraper { * @returns {Promise} */ async go(location = 'london') { - this.setStartUrl(`https://www.totaljobs.com/jobs/contract/html-or-vue-or-vuejs-or-web-or-sql-or-delphi-or-vb-or-vbscript-or-php-or-ajax-or-mysql-or-sqlserver-or-javascript-or-node-or-nodejs-or-svelte-or-sveltejs-not-react/in-${encodeURIComponent(location)}?q=Html+Or+Vue+Or+Vuejs+Or+Web+Or+Sql+Or+Delphi+Or+Vb+Or+Vbscript+Or+Php+Or+Ajax+Or+Mysql+Or+Sqlserver+Or+Javascript+Or+Node+Or+Nodejs+Or+Svelte+Or+Sveltejs+NOT+React&postedwithin=3&radius=20`); + this.setStartUrl(`https://www.totaljobs.com/jobs/contract/html-or-angular-or-vue-or-vuejs-or-web-or-sql-or-delphi-or-vb-or-vbscript-or-php-or-ajax-or-mysql-or-sqlserver-or-javascript-or-node-or-nodejs-or-svelte-or-sveltejs-not-react/in-${encodeURIComponent(location)}?q=Html+Or+Vue+Or+Vuejs+Or+Web+Or+Sql+Or+Delphi+Or+Vb+Or+Vbscript+Or+Php+Or+Ajax+Or+Mysql+Or+Sqlserver+Or+Javascript+Or+Node+Or+Nodejs+Or+Svelte+Or+Sveltejs+NOT+React&postedwithin=3&radius=20`); // this.setStartUrl('https://www.indeed.co.uk/jobs?as_and=&as_phr=&as_any=javascript+nodejs&as_not=&as_ttl=&as_cmp=&jt=contract&st=&as_src=&salary=&radius=25&l=london&fromage=7&limit=10&sort=date&psf=advsrch&from=advancedsearch'); // Glasgow