jobscraper/scrapers/indeed.js

152 lines
3.6 KiB
JavaScript
Raw Normal View History

2020-05-19 09:05:04 +00:00
/**
* Created by WebStorm.
* User: martin
* Date: 15/04/2020
* Time: 11:55
*/
const cheerio = require('cheerio');
const MasterScraper = require('../lib/scraper');
class IndeedScraper extends MasterScraper {
2020-05-19 09:05:04 +00:00
constructor() {
super();
this.siteurl = 'www.indeed.co.uk';
this.siteid = 'indeed';
this.useStone = true;
this.requestOptions = {
'url' : ''
};
this.antiAd = /sja\d+/gi;
}
2020-05-19 09:05:04 +00:00
// Site specific parts below here
2020-05-19 09:05:04 +00:00
async breakPage() {
const $ = this.currentPage;
const ads = [];
2020-05-19 09:05:04 +00:00
const sections = $('div.row.result');
2020-05-19 09:05:04 +00:00
await sections.each(async (index, item) => {
// console.log($(item).html());
const ad = await this.extractDetails(item);
if (ad !== null)
ads.push(ad);
console.log(ads);
// console.log('<<<<<<<<<>>>>>>>>>');
});
this.items = [...this.items, ...ads];
}
async extractDetails(part) {
const newObj = {};
const $part = cheerio.load(part);
// console.log($part.html());
const now = ~~(new Date().getTime() / 1000.0);
newObj.title = $part('.jobtitle').text().trim();
newObj.site = this.siteid;
// newObj.url = `https://${ this.siteurl }${$part('.jobtitle').attr('href')}`;
newObj.url = this.makeUrl($part('.jobtitle').attr('href'));
newObj.id = $part('h2.title a').attr('id').trim();
newObj.summary = $part('.summary').text().trim();
newObj.company = $part('.company').text().trim() || null;
newObj.location = $part('.location').text().trim();
newObj.postDate = $part('.date').text().trim();
newObj.salary = $part('.salary.no-wrap').text().trim();
newObj.isEasyApply = $part('.iaLabel').text().trim() === 'Easily apply';
newObj.timestamp = now;
return newObj;
}
2020-05-19 09:05:04 +00:00
async getIndividualPage(item) {
const newItem = {...item};
console.log('Getting', item.url);
await this.getContent(item.url)
.then((html) => {
const $ = cheerio.load(html);
2020-05-22 22:40:33 +00:00
newItem.summary = $('#jobDescriptionText').html();
2020-05-19 09:05:04 +00:00
})
.catch((err) => console.error(err));
return newItem;
}
async getJobPages() {
const newItems = [];
for (let item of this.items) {
item = await this.getIndividualPage(item);
newItems.push(item);
}
this.items = [...newItems];
}
async checkNext() {
const $ = this.currentPage;
const next = $('.pagination > *:last-child').attr('href') || '';
if (next !== '')
2020-05-19 09:05:04 +00:00
// next = `https://${ this.siteurl }${next}`;
this.makeUrl(next);
2020-05-19 09:05:04 +00:00
console.log(next);
}
async processSite() {
console.log('Processing...');
let nextPage;
const previousPage = '';
// do {
// previousPage = this.url;
this.items = [];
await this.getPage();
await this.breakPage();
await this.checkNext();
await this.getJobPages();
2020-05-19 09:05:04 +00:00
// nextPage = await this.checkNext();
2020-05-19 09:05:04 +00:00
// if (nextPage === previousPage) nextPage = '';
// this.setStartUrl(nextPage);
// }while (nextPage !== '');
await this.filterAdverts();
await this.addToDB();
2020-08-24 08:35:30 +00:00
await this.addToMongo();
2020-05-19 09:05:04 +00:00
}
async go(location = 'london') {
this.setStartUrl(`https://www.indeed.co.uk/jobs?as_and=&as_phr=&as_any=Angular+Html+Web+Sql+Delphi+Vb+Vbscript+Php+Ajax+Mysql+Sqlserver+Javascript+Nodejs+vuejs+sveltejs&as_not=React&as_ttl=&as_cmp=&jt=contract&st=&as_src=&salary=&radius=0&l=${encodeURIComponent(location)}&fromage=1&limit=50&sort=&psf=advsrch&from=advancedsearch`);
2020-05-19 09:05:04 +00:00
await this.processSite().catch((err) => {
console.error('Indeed Go', err);
});
2020-06-01 08:25:13 +00:00
console.log(`Indeed ${location} completed`);
2020-05-19 09:05:04 +00:00
}
2020-05-19 09:05:04 +00:00
}
module.exports = IndeedScraper;