martin/jobscraper
martin/jobscraper
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

added scraper test

Browse Source
This commit is contained in:
Martin Donnelly 2020-06-01 09:25:13 +01:00
parent b43a54c0e4
commit da03107d96
4 changed files with 6 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
scrapers/cwjobs.js
View File

@ -27,6 +27,7 @@ class CwjobsScraper extends TotaljobsScraper {
// https://www.indeed.co.uk/jobs?as_and=&as_phr=&as_any=Html+Web+Sql+Delphi+Vb+Vbscript+Php+Ajax+Mysql+Sqlserver+Javascript+Nodejs+vuejs+sveltejs&as_not=React&as_ttl=&as_cmp=&jt=contract&st=&as_src=&salary=&radius=0&l=glasgow&fromage=1&limit=50&sort=&psf=advsrch&from=advancedsearch
await this.processSite();
console.log(`CWJobs ${location} completed`);
}
}

2
scrapers/indeed.js
View File

@ -143,6 +143,8 @@ class IndeedScraper extends MasterScraper {
// https://www.indeed.co.uk/jobs?as_and=&as_phr=&as_any=Html+Web+Sql+Delphi+Vb+Vbscript+Php+Ajax+Mysql+Sqlserver+Javascript+Nodejs+vuejs+sveltejs&as_not=React&as_ttl=&as_cmp=&jt=contract&st=&as_src=&salary=&radius=0&l=glasgow&fromage=1&limit=50&sort=&psf=advsrch&from=advancedsearch
await this.processSite();
console.log(`Indeed ${location} completed`);
}
}

1
scrapers/totaljobs.js
View File

@ -131,6 +131,7 @@ class TotaljobsScraper extends MasterScraper {
// https://www.indeed.co.uk/jobs?as_and=&as_phr=&as_any=Html+Web+Sql+Delphi+Vb+Vbscript+Php+Ajax+Mysql+Sqlserver+Javascript+Nodejs+vuejs+sveltejs&as_not=React&as_ttl=&as_cmp=&jt=contract&st=&as_src=&salary=&radius=0&l=glasgow&fromage=1&limit=50&sort=&psf=advsrch&from=advancedsearch
await this.processSite();
console.log(`TotalJobs ${location} completed`);
}
}

17
test/indeed.js
View File

@ -21,7 +21,7 @@ const indeedScraper = new IndeedScraper();
const page = fs.readFileSync('data/indeed/page2.html');
test.test('Test Indeed scraper', async t => {
/*const $ = cheerio.load(page);
const $ = cheerio.load(page);
indeedScraper.loadPage($);
@ -35,20 +35,7 @@ test.test('Test Indeed scraper', async t => {
await indeedScraper.filterAdverts();
// await indeedScraper.addToDB();*/
await indeedScraper.go('london');
await totaljobsScraper.go('london');
await cwjobsScraper.go('london');
await indeedScraper.go('glasgow');
await totaljobsScraper.go('glasgow');
await cwjobsScraper.go('glasgow');
await indeedScraper.go('edinburgh');
await totaljobsScraper.go('edinburgh');
await cwjobsScraper.go('edinburgh');
await indeedScraper.go('milton keynes');
await totaljobsScraper.go('milton keynes');
await cwjobsScraper.go('milton keynes');
// await indeedScraper.addToDB();
t.end();
});