111 lines
1.9 KiB
JavaScript
111 lines
1.9 KiB
JavaScript
/**
|
|
* Created by WebStorm.
|
|
* User: martin
|
|
* Date: 22/05/2020
|
|
* Time: 12:01
|
|
|
|
*/
|
|
const Parser = require('rss-parser');
|
|
let parser;
|
|
|
|
const MasterBase = require('./base');
|
|
|
|
const isEmpty = obj => [Object, Array].includes((obj || {}).constructor) && !Object.entries((obj || {})).length;
|
|
|
|
class MasterRSS extends MasterBase {
|
|
|
|
constructor(options = {}) {
|
|
super();
|
|
|
|
parser = new Parser(options);
|
|
|
|
this.rawItems = [];
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param file
|
|
* @returns {Promise<void>}
|
|
*/
|
|
async loadFeed(file) {
|
|
console.log('>> retrieveFeed');
|
|
const feed = await parser.parseString(file).catch((err) => {
|
|
console.error(err);
|
|
});
|
|
|
|
console.log(feed);
|
|
|
|
this.rawItems = [...feed.items];
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @returns {Promise<void>}
|
|
*/
|
|
async retrieveFeed() {
|
|
console.log('>> retrieveFeed');
|
|
let _url;
|
|
|
|
if (this.useStone)
|
|
_url = `http://45.33.114.116:8080/${encodeURIComponent(this.url)}`;
|
|
else
|
|
_url = this.url;
|
|
|
|
console.log(_url);
|
|
await parser.parseURL(_url)
|
|
.then((feed) => {
|
|
// console.log(feed);
|
|
|
|
this.rawItems = [...feed.items];
|
|
console.log(`${this.url} // ${this.rawItems.length}`);
|
|
})
|
|
.catch((err) => {
|
|
console.error('retrieveFeed', err);
|
|
});
|
|
}
|
|
|
|
/**
|
|
*
|
|
*/
|
|
reduceItems() {
|
|
if(this.rawItems.length > 0)
|
|
this.rawItems.forEach(item => {
|
|
this.items.push(this.reduceItem(item));
|
|
});
|
|
|
|
// console.log(this.items);
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @returns {Promise<void>}
|
|
*/
|
|
async processFeed() {
|
|
await this.retrieveFeed();
|
|
|
|
if(this.rawItems.length > 0) {
|
|
this.reduceItems();
|
|
|
|
await this.filterAdverts();
|
|
|
|
if (this.items.length > 0) await this.addToDB();
|
|
if (this.items.length > 0) await this.addToMongo();
|
|
}
|
|
else
|
|
console.log('No items to process');
|
|
}
|
|
|
|
// -------------
|
|
|
|
/**
|
|
*
|
|
* @param item
|
|
*/
|
|
reduceItem(item) {
|
|
|
|
}
|
|
|
|
}
|
|
|
|
module.exports = MasterRSS;
|