obdfcascrape/tests/scrape.cy.js

const tape = require('tape');
const _test = require('tape-promise').default; // <---- notice 'default'
const test = _test(tape); // decorate tape

const Cyprus = require('../ncas/cy');

const fs = require('fs');
const jsonfile = require('jsonfile');

test.skip('CYPRUS::Scraper', async t => {
  const cyScraper = new Cyprus();

  await cyScraper._initBrowser();

  cyScraper.browser.on('response', async target => {
    console.log('response', target);
  });

  cyScraper.browser.on('targetcreated', async target => {
    console.log(target);
    if (target.url() !== 'about:blank') {
      const tsDirectory = await this._createTimestampDirectory(this.path);

      await cyScraper.page._client.send('Page.setDownloadBehavior', { 'behavior': 'allow', 'downloadPath': tsDirectory });
      console.log('Saving into:', tsDirectory);
    }
  });

  cyScraper.page = await cyScraper.browser.newPage();

  await cyScraper.page.goto('http://127.0.0.1:8080/test.html', { 'waitUntil': 'networkidle0' });

  await cyScraper.downloadExcel();

  t.deepEquals({}, {}, 'Extract the correct data');
  t.end();
});

test('CYPRUS:: Scrape a Credit Service', async t => {
  t.test('CYPRUS::Extract Local Credit Institutions from Page', async t => {
    const contentDetail = await fs.readFileSync('tests/data/cy/content.html');
    const expectedJSON = jsonfile.readFileSync('tests/data/cy/local_credit_001.json');

    const cyScraper = new Cyprus();

    const output = await cyScraper.extractLocalCreditInstitutions(contentDetail);

    t.deepEquals(output, expectedJSON, 'Extracted Local Credit Institutions from Page');

    t.end();
  });

  t.test('CYPRUS::Extract Foreign Credit Institutions from Page', async t => {
    const contentDetail = fs.readFileSync('tests/data/cy/content.html');
    const expectedJSON = jsonfile.readFileSync('tests/data/cy/foreign_credit_001.json');

    const cyScraper = new Cyprus();

    const output = await cyScraper.extractForeignCreditInstitutions(contentDetail);

    t.deepEquals(output, expectedJSON, 'Extracted Foreign Credit Institutions from Page');

    t.end();
  });

  t.skip('CYPRUS::Extract ALL Credit Institutions from Page', async t => {
    const contentDetail = fs.readFileSync('tests/data/cy/credit_institutes.html');
    const expectedJSON = jsonfile.readFileSync('tests/data/cy/all_credit_001.json');

    const cyScraper = new Cyprus();

    cyScraper.path = 'tests/sink';
    await cyScraper._initBrowser(false);
    cyScraper.page = await cyScraper.browser.newPage();

    await cyScraper.page.goto(`data:text/html,${contentDetail}`, { 'waitUntil': 'networkidle0' });

    const output = await cyScraper.processCreditInstitute();
    await cyScraper.browser.close();

    t.deepEquals(output, expectedJSON, 'Extracted ALL Credit Institutions from Page');

    t.end();
  });
});