69 lines
1.4 KiB
JavaScript
69 lines
1.4 KiB
JavaScript
/**
|
|
* Created by WebStorm.
|
|
* User: martin
|
|
* Date: 15/04/2020
|
|
* Time: 11:56
|
|
|
|
*/
|
|
|
|
const tape = require('tape');
|
|
const _test = require('tape-promise').default; // <---- notice 'default'
|
|
const test = _test(tape); // decorate tape
|
|
|
|
const fs = require('fs');
|
|
const cheerio = require('cheerio');
|
|
|
|
const IndeedScraper = require('../scrapers/indeed');
|
|
|
|
const indeedScraper = new IndeedScraper();
|
|
|
|
// const page = fs.readFileSync('data/indeed/indeed-2020-04-16--092311.html');
|
|
const page = fs.readFileSync('data/indeed/page2.html');
|
|
|
|
test.skip('Test Indeed scraper', async t => {
|
|
const $ = cheerio.load(page);
|
|
|
|
indeedScraper.loadPage($);
|
|
|
|
await indeedScraper.breakPage();
|
|
|
|
// await indeedScraper.getJobPages();
|
|
|
|
// console.log(await indeedScraper.checkNext());
|
|
|
|
// console.log(indeedScraper.items);
|
|
|
|
await indeedScraper.filterAdverts();
|
|
|
|
await indeedScraper.addToMongo();
|
|
|
|
t.end();
|
|
});
|
|
|
|
test.skip('Test full run Indeed scraper', async t => {
|
|
await indeedScraper.go('london');
|
|
|
|
t.end();
|
|
});
|
|
|
|
|
|
test.test('Test Indeed scraper -- MONGO', async t => {
|
|
const $ = cheerio.load(page);
|
|
|
|
indeedScraper.loadPage($);
|
|
|
|
await indeedScraper.breakPage();
|
|
|
|
// await indeedScraper.getJobPages();
|
|
|
|
// console.log(await indeedScraper.checkNext());
|
|
|
|
// console.log(indeedScraper.items);
|
|
|
|
// await indeedScraper.filterAdverts();
|
|
|
|
await indeedScraper.addToMongo();
|
|
|
|
t.end();
|
|
});
|