cc-tracking/tasks/fresh Scraper.md

Fresh Scraper
========================


```javascript



const Scraper = require('../helpers/scraper');
const cheerio = require('cheerio');
const path = require('path');
const jsonfile = require('jsonfile');
const logger = require('log4js').getLogger('LV');
const url = require('url');

logger.level = process.env.LOGGER_LEVEL || 'warn';

class LVScrape extends Scraper {

  constructor() {
    super();
    this.id = 'LV';

    this.on('done', () => {
      this._done();
    });

    this.run = this._throttle(async () => {
      await this.__run();
    }, 5000);

    if (process.env.NODE_ENV === 'production')
      this._checkLock().then((l) => {
        if(l)
          this.run();
      });
  }

  async processNewPage() {
    // give the page a few seconds to settle
    await this._randomWait(this.page, 3, 5);

    const pageUrl = url.parse(await this.page.url());

    switch (pageUrl.pathname) {

      case '/en/our-registers/company-register/':
        await this.indexRedirector();
        break;

      case '/en/our-registers/company-register/details':
        await this.processRedirector();
        break;
      case '/en/our-registers/company-register/gransoverskridandehandel/':
        await this.crossBorderRedirector();
        break;

      default:
        if (process.env.NODE_ENV) {
          await this._uploadError();
          throw new Error(`Unknown page: ${pageUrl}`);
        }
        else {
          logger.warn('processNewPage Fell through');
          logger.warn('currentPage.location', pageUrl);
        }
        break;

    }
  }

  async start() {
    super._start();
    try {
      this.mode = 0;

      this.paymentServices = {
        'items': 0,
        'links': [],
        'step': 0,
        'indexStep': 0,
        'visited': false,
        'done' : false,
        'urls': ['https://www.mfsa.com.mt/pages/licenceholders.aspx'],
        'sections' : [],
        'sectionLinks' : []
      };

      this.emoneyServices = {
        'items': 0,
        'links': [],
        'step': 0,
        'indexStep': 0,
        'visited': false,
        'done' : false,
        'urls': [],
        'sections' : [],
        'sectionLinks' : []
      };

      this.creditServices = {
        'items': 0,
        'links': [],
        'step': 0,
        'indexStep': 0,
        'visited': false,
        'done' : false,
        'searchDone' : false,
        'started': false,
        'urls': [],
        'sections' : [],
        'sectionLinks' : []
      };

      this.startPage = this.paymentServices.urls[0];
      this.emoneyUrl = this.emoneyServices.urls[0];
      this.credit = this.creditServices.urls[0];

      this.setPath(path.resolve(`${__dirname }/../artefacts/MT/MFSA`));

      // await this._doNonRepudiation();

      await this._initBrowser(false);
      this.page = await this.browser.newPage();

      this.page.on('domcontentloaded', () => {
        this.processNewPage();
      });

      this.on('entityComplete', () => {
        this.handleEntityComplete();
      });

      this.on('serviceDone', async function() {
        switch (this.mode) {

          case 0:
            this.emit('paymentServicesDone');
            break;

          case 1:
            this.emit('emoneyServicesDone');
            break;

          case 2:
            this.emit('creditServicesDone');
            break;

        }
      });

      this.on('paymentServicesDone', async function() {
        logger.warn('paymentServicesDone');
        try{
          this.paymentServices.done = true;
          jsonfile.writeFileSync(`${this.path}/paymentServices.json`, { 'links': this.paymentServices.links });
          jsonfile.writeFileSync(`${this.debugPath}/paymentServices.json`, this.paymentServices);

          this.mode++;
          this.inProgress = false;

          await this._goto(this.emoneyServices.urls[0]);
        }
        catch (e) {
          logger.error(e);
        }
      });

      this.on('emoneyServicesDone', async function() {
        logger.warn('emoneyServicesDone');
        try{
          this.emoneyServices.done = true;
          jsonfile.writeFileSync(`${this.path}/emoneyServices.json`, { 'links':this.emoneyServices.links });
          jsonfile.writeFileSync(`${this.debugPath}/emoneyServices.json`, this.emoneyServices);
          this.mode++;
          this.inProgress = false;

          await this._goto(this.creditServices.urls[0]);
        }
        catch (e) {
          logger.error(e);
        }
      });

      this.on('creditServicesDone', async function() {
        logger.warn('creditServicesDone');
        try{
          this.creditServices.done = true;
          jsonfile.writeFileSync(`${this.path}/creditServices.json`, { 'links':this.creditServices.links });
          jsonfile.writeFileSync(`${this.debugPath}/creditServices.json`, this.creditServices);
          this.mode++;
          this.inProgress = false;

          this.emit('done');
        }
        catch (e) {
          logger.error(e);
        }
      });

      //

      await this.page.setViewport({ 'width': 1200, 'height': 800 });
      await this._goto(this.startPage, { 'waitUntil':'networkidle0' });

      await this._randomWait(this.page, 3, 5);
    }
    catch(e) {
      throw new Error(e);
    }
  }

  async __run() {
    await this.start();
  }

}

module.exports = LVScrape;








```