jobscraper/lib/base.js

/**
 * Created by WebStorm.
 * User: martin
 * Date: 22/05/2020
 * Time: 12:01

 */
const filterReject = require('../lib/filter_reject');
const filterAccept = require('../lib/filter_md_jobs');
const dbmanager = require('../lib/dbmanager');

class MasterBase {

  /**
   *
   */
  constructor() {
    this.url = '';
    this.items = [];
    this.currentPage = null;
    this.hosturl = '';
    this.siteid = '';
    this.useStone = false;
    this.saveFile = false;
    this.requestOptions = {
      'url'    : '',
      'proxy'  : 'http://uk.proxymesh.com:31280',
      'tunnel' : true
    };
  }

  /**
   *
   * @returns {{summary: string, site: string, postDate: string, location: string, company: string, id: string, title: string, isEasyApply: boolean, salary: string, url: string, timestamp: number}}
   */
  newRecord() {
    const now = ~~(new Date().getTime() / 1000.0);

    return { 'title': '', 'site': this.siteid || '', 'url':'', 'id':'', 'summary':'', 'postDate':'', 'isEasyApply':false, 'location': '', 'company': '', 'salary': '', 'timestamp':now };
  }

  /**
   * 
   * @returns {Promise<void>}
   */
  async addToDB() {
    for(const item of this.items) 
    // console.log(item);

      dbmanager.insertOne(item)
        .then((data) => {
          console.log(data);
        })
        .catch((err) => {
          console.error(`${this.siteid} db error`);
          console.error(err.message || 'Some error occurred while querying the database.');
        });
  }

  /**
   * 
   * @returns {Promise<void>}
   */
  async filterAdverts() {
    console.log('>> FilterAdverts');
    console.log(`Currently ${this.items.length} items...`);

    this.items = this.items.filter(filterReject);

    console.log(`After reject ${this.items.length} items...`);

    this.items = this.items.filter(filterAccept);

    console.log(`After accept ${this.items.length} items...`);

    //  console.log(this.items);
  }
  
  /**
   *
   * @param newUrl
   */
  setStartUrl(newUrl) {
    this.url = newUrl;
  }

  /**
   *
   * @param page
   */
  loadPage(page) {
    this.currentPage = page;
  }

  /**
   *
   * @param appended
   * @returns {string}
   */
  makeUrl(appended) {
    return `https://${ this.siteurl }${appended}`;
  }

  /**
   *
   * @param appended
   * @returns {string}
   */
  makeProxyUrl(appended) {
    return `https://${ this.siteurl }${appended}`;
  }

  /**
   *
   * @param url
   * @param q
   * @returns {string}
   */
  makeImg(url, q = 75) {
    return `https://image.silvrtree.co.uk/q${q}/${url}`;
  }

  async go() {
    this.items = [];
    this.rawItems = [];
  }
}

module.exports = MasterBase;