128 lines
2.5 KiB
JavaScript
128 lines
2.5 KiB
JavaScript
/**
|
|
* Created by WebStorm.
|
|
* User: martin
|
|
* Date: 22/05/2020
|
|
* Time: 12:01
|
|
|
|
*/
|
|
const filterReject = require('../lib/filter_reject');
|
|
const filterAccept = require('../lib/filter_md_jobs');
|
|
const dbmanager = require('../lib/dbmanager');
|
|
|
|
class MasterBase {
|
|
|
|
/**
|
|
*
|
|
*/
|
|
constructor() {
|
|
this.url = '';
|
|
this.items = [];
|
|
this.currentPage = null;
|
|
this.hosturl = '';
|
|
this.siteid = '';
|
|
this.useStone = false;
|
|
this.requestOptions = {
|
|
'url' : '',
|
|
'proxy' : 'http://uk.proxymesh.com:31280',
|
|
'tunnel' : true
|
|
};
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @returns {{summary: string, site: string, postDate: string, location: string, company: string, id: string, title: string, isEasyApply: boolean, salary: string, url: string, timestamp: number}}
|
|
*/
|
|
newRecord() {
|
|
const now = ~~(new Date().getTime() / 1000.0);
|
|
|
|
return { 'title': '', 'site': this.siteid || '', 'url':'', 'id':'', 'summary':'', 'postDate':'', 'isEasyApply':false, 'location': '', 'company': '', 'salary': '', 'timestamp':now };
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @returns {Promise<void>}
|
|
*/
|
|
async addToDB() {
|
|
for(const item of this.items)
|
|
// console.log(item);
|
|
|
|
dbmanager.insertOne(item)
|
|
.then((data) => {
|
|
console.log(data);
|
|
})
|
|
.catch((err) => {
|
|
console.error(err.message || 'Some error occurred while querying the database.');
|
|
});
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @returns {Promise<void>}
|
|
*/
|
|
async filterAdverts() {
|
|
console.log('>> FilterAdverts');
|
|
console.log(`Currently ${this.items.length} items...`);
|
|
|
|
this.items = this.items.filter(filterReject);
|
|
|
|
console.log(`After reject ${this.items.length} items...`);
|
|
|
|
this.items = this.items.filter(filterAccept);
|
|
|
|
console.log(`After accept ${this.items.length} items...`);
|
|
|
|
// console.log(this.items);
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param newUrl
|
|
*/
|
|
setStartUrl(newUrl) {
|
|
this.url = newUrl;
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param page
|
|
*/
|
|
loadPage(page) {
|
|
this.currentPage = page;
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param appended
|
|
* @returns {string}
|
|
*/
|
|
makeUrl(appended) {
|
|
return `https://${ this.siteurl }${appended}`;
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param appended
|
|
* @returns {string}
|
|
*/
|
|
makeProxyUrl(appended) {
|
|
return `https://${ this.siteurl }${appended}`;
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param url
|
|
* @param q
|
|
* @returns {string}
|
|
*/
|
|
makeImg(url, q = 75) {
|
|
return `https://image.silvrtree.co.uk/q${q}/${url}`;
|
|
}
|
|
|
|
async go() {
|
|
this.items = [];
|
|
this.rawItems = [];
|
|
}
|
|
}
|
|
|
|
module.exports = MasterBase;
|