jobscraper/test/indeed.js

/**
 * Created by WebStorm.
 * User: martin
 * Date: 15/04/2020
 * Time: 11:56

 */

const tape = require('tape');
const _test = require('tape-promise').default; // <---- notice 'default'
const test = _test(tape); // decorate tape

const fs = require('fs');
const cheerio = require('cheerio');

const IndeedScraper = require('../scrapers/indeed');

const indeedScraper = new IndeedScraper();

// const page = fs.readFileSync('data/indeed/indeed-2020-04-16--092311.html');
const page = fs.readFileSync('data/indeed/page2.html');

test.skip('Test Indeed scraper', async t => {
  const $ = cheerio.load(page);

  indeedScraper.loadPage($);

  await indeedScraper.breakPage();

  // await indeedScraper.getJobPages();

  // console.log(await indeedScraper.checkNext());

  // console.log(indeedScraper.items);

  await indeedScraper.filterAdverts();

  await indeedScraper.addToMongo();

  t.end();
});

test.skip('Test full run Indeed scraper', async t => {
  await indeedScraper.go('london').catch((err) => {
    console.error('Indeed GO', err);
  });
  
  t.end();
});


test.test('Test Indeed scraper -- MONGO', async t => {
  const $ = cheerio.load(page);

  indeedScraper.loadPage($);

  await indeedScraper.breakPage();

  // await indeedScraper.getJobPages();

  // console.log(await indeedScraper.checkNext());

  // console.log(indeedScraper.items);

 // await indeedScraper.filterAdverts();

  await indeedScraper.addToMongo();

  t.end();
});