martin/jobscraper
martin/jobscraper
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added angular to indeed, cws and total

Browse Source 
Added a new search for jobserve
This commit is contained in:
Martin Donnelly 2020-12-10 16:05:21 +00:00
parent c4713863c0
commit 17a348f625
4 changed files with 15 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
grabber.js
View File

@ -38,6 +38,7 @@ const RssTechnojobs = require('./scrapers/rss.technojobs');
}, null, true); }, null, true);
new CronJob('0 6-23/1 * * *', async function() { new CronJob('0 6-23/1 * * *', async function() {
await jobserveScraper.go('https://www.jobserve.com/MySearch/D48462060FB24B6C.rss');
await jobserveScraper.go('https://www.jobserve.com/MySearch/BAEBF3BDF82B8FEF.rss'); await jobserveScraper.go('https://www.jobserve.com/MySearch/BAEBF3BDF82B8FEF.rss');
await jobserveScraper.go('https://www.jobserve.com/MySearch/9BCBF25C586A0E3F.rss'); await jobserveScraper.go('https://www.jobserve.com/MySearch/9BCBF25C586A0E3F.rss');
await jobserveScraper.go('https://www.jobserve.com/MySearch/F3A56475D5FD4966.rss'); await jobserveScraper.go('https://www.jobserve.com/MySearch/F3A56475D5FD4966.rss');

2
scrapers/cwjobs.js
View File

@ -20,7 +20,7 @@ class CwjobsScraper extends TotaljobsScraper {
} }
async go(location = 'london') { async go(location = 'london') {
this.setStartUrl(`https://www.cwjobs.co.uk/jobs/contract/html-or-vue-or-vuejs-or-web-or-sql-or-delphi-or-vb-or-vbscript-or-php-or-ajax-or-mysql-or-sqlserver-or-javascript-or-node-or-nodejs-or-svelte-or-sveltejs-not-react/in-${encodeURIComponent(location)}?q=Html+Or+Vue+Or+Vuejs+Or+Web+Or+Sql+Or+Delphi+Or+Vb+Or+Vbscript+Or+Php+Or+Ajax+Or+Mysql+Or+Sqlserver+Or+Javascript+Or+Node+Or+Nodejs+Or+Svelte+Or+Sveltejs+NOT+React&postedwithin=3&radius=20`); this.setStartUrl(`https://www.cwjobs.co.uk/jobs/contract/html-or-angular-or-vue-or-vuejs-or-web-or-sql-or-delphi-or-vb-or-vbscript-or-php-or-ajax-or-mysql-or-sqlserver-or-javascript-or-node-or-nodejs-or-svelte-or-sveltejs-not-react/in-${encodeURIComponent(location)}?q=Html+Or+Vue+Or+Vuejs+Or+Web+Or+Sql+Or+Delphi+Or+Vb+Or+Vbscript+Or+Php+Or+Ajax+Or+Mysql+Or+Sqlserver+Or+Javascript+Or+Node+Or+Nodejs+Or+Svelte+Or+Sveltejs+NOT+React&postedwithin=3&radius=20`);
// this.setStartUrl('https://www.indeed.co.uk/jobs?as_and=&as_phr=&as_any=javascript+nodejs&as_not=&as_ttl=&as_cmp=&jt=contract&st=&as_src=&salary=&radius=25&l=london&fromage=7&limit=10&sort=date&psf=advsrch&from=advancedsearch'); // this.setStartUrl('https://www.indeed.co.uk/jobs?as_and=&as_phr=&as_any=javascript+nodejs&as_not=&as_ttl=&as_cmp=&jt=contract&st=&as_src=&salary=&radius=25&l=london&fromage=7&limit=10&sort=date&psf=advsrch&from=advancedsearch');
// Glasgow // Glasgow

24
scrapers/indeed.js
View File

@ -11,7 +11,7 @@ const cheerio = require('cheerio');
const MasterScraper = require('../lib/scraper'); const MasterScraper = require('../lib/scraper');
class IndeedScraper extends MasterScraper { class IndeedScraper extends MasterScraper {
constructor() { constructor() {
super(); super();
this.siteurl = 'www.indeed.co.uk'; this.siteurl = 'www.indeed.co.uk';
@ -23,15 +23,15 @@ class IndeedScraper extends MasterScraper {
this.antiAd = /sja\d+/gi; this.antiAd = /sja\d+/gi;
} }
// Site specific parts below here // Site specific parts below here
async breakPage() { async breakPage() {
const $ = this.currentPage; const $ = this.currentPage;
const ads = []; const ads = [];
const sections = $('div.row.result'); const sections = $('div.row.result');
await sections.each(async (index, item) => { await sections.each(async (index, item) => {
// console.log($(item).html()); // console.log($(item).html());
const ad = await this.extractDetails(item); const ad = await this.extractDetails(item);
@ -73,7 +73,7 @@ class IndeedScraper extends MasterScraper {
return newObj; return newObj;
} }
async getIndividualPage(item) { async getIndividualPage(item) {
const newItem = {...item}; const newItem = {...item};
console.log('Getting', item.url); console.log('Getting', item.url);
@ -100,10 +100,10 @@ class IndeedScraper extends MasterScraper {
async checkNext() { async checkNext() {
const $ = this.currentPage; const $ = this.currentPage;
const next = $('.pagination > *:last-child').attr('href') || ''; const next = $('.pagination > *:last-child').attr('href') || '';
if (next !== '') if (next !== '')
// next = `https://${ this.siteurl }${next}`; // next = `https://${ this.siteurl }${next}`;
this.makeUrl(next); this.makeUrl(next);
console.log(next); console.log(next);
} }
@ -122,9 +122,9 @@ class IndeedScraper extends MasterScraper {
await this.checkNext(); await this.checkNext();
await this.getJobPages(); await this.getJobPages();
// nextPage = await this.checkNext(); // nextPage = await this.checkNext();
// if (nextPage === previousPage) nextPage = ''; // if (nextPage === previousPage) nextPage = '';
// this.setStartUrl(nextPage); // this.setStartUrl(nextPage);
@ -137,7 +137,7 @@ class IndeedScraper extends MasterScraper {
} }
async go(location = 'london') { async go(location = 'london') {
this.setStartUrl(`https://www.indeed.co.uk/jobs?as_and=&as_phr=&as_any=Html+Web+Sql+Delphi+Vb+Vbscript+Php+Ajax+Mysql+Sqlserver+Javascript+Nodejs+vuejs+sveltejs&as_not=React&as_ttl=&as_cmp=&jt=contract&st=&as_src=&salary=&radius=0&l=${encodeURIComponent(location)}&fromage=1&limit=50&sort=&psf=advsrch&from=advancedsearch`); this.setStartUrl(`https://www.indeed.co.uk/jobs?as_and=&as_phr=&as_any=Angular+Html+Web+Sql+Delphi+Vb+Vbscript+Php+Ajax+Mysql+Sqlserver+Javascript+Nodejs+vuejs+sveltejs&as_not=React&as_ttl=&as_cmp=&jt=contract&st=&as_src=&salary=&radius=0&l=${encodeURIComponent(location)}&fromage=1&limit=50&sort=&psf=advsrch&from=advancedsearch`);
await this.processSite().catch((err) => { await this.processSite().catch((err) => {
console.error('Indeed Go', err); console.error('Indeed Go', err);
@ -145,7 +145,7 @@ class IndeedScraper extends MasterScraper {
console.log(`Indeed ${location} completed`); console.log(`Indeed ${location} completed`);
} }
} }
module.exports = IndeedScraper; module.exports = IndeedScraper;

2
scrapers/totaljobs.js
View File

@ -155,7 +155,7 @@ class TotaljobsScraper extends MasterScraper {
* @returns {Promise<void>} * @returns {Promise<void>}
*/ */
async go(location = 'london') { async go(location = 'london') {
this.setStartUrl(`https://www.totaljobs.com/jobs/contract/html-or-vue-or-vuejs-or-web-or-sql-or-delphi-or-vb-or-vbscript-or-php-or-ajax-or-mysql-or-sqlserver-or-javascript-or-node-or-nodejs-or-svelte-or-sveltejs-not-react/in-${encodeURIComponent(location)}?q=Html+Or+Vue+Or+Vuejs+Or+Web+Or+Sql+Or+Delphi+Or+Vb+Or+Vbscript+Or+Php+Or+Ajax+Or+Mysql+Or+Sqlserver+Or+Javascript+Or+Node+Or+Nodejs+Or+Svelte+Or+Sveltejs+NOT+React&postedwithin=3&radius=20`); this.setStartUrl(`https://www.totaljobs.com/jobs/contract/html-or-angular-or-vue-or-vuejs-or-web-or-sql-or-delphi-or-vb-or-vbscript-or-php-or-ajax-or-mysql-or-sqlserver-or-javascript-or-node-or-nodejs-or-svelte-or-sveltejs-not-react/in-${encodeURIComponent(location)}?q=Html+Or+Vue+Or+Vuejs+Or+Web+Or+Sql+Or+Delphi+Or+Vb+Or+Vbscript+Or+Php+Or+Ajax+Or+Mysql+Or+Sqlserver+Or+Javascript+Or+Node+Or+Nodejs+Or+Svelte+Or+Sveltejs+NOT+React&postedwithin=3&radius=20`);
// this.setStartUrl('https://www.indeed.co.uk/jobs?as_and=&as_phr=&as_any=javascript+nodejs&as_not=&as_ttl=&as_cmp=&jt=contract&st=&as_src=&salary=&radius=25&l=london&fromage=7&limit=10&sort=date&psf=advsrch&from=advancedsearch'); // this.setStartUrl('https://www.indeed.co.uk/jobs?as_and=&as_phr=&as_any=javascript+nodejs&as_not=&as_ttl=&as_cmp=&jt=contract&st=&as_src=&salary=&radius=25&l=london&fromage=7&limit=10&sort=date&psf=advsrch&from=advancedsearch');
// Glasgow // Glasgow