const RssBraider = require('rss-braider'); const fs = require('fs'); const util = require('util'); const fsWriteFile = util.promisify(fs.writeFile); const feeds = {}; const log4js = require('log4js'); const logger = log4js.getLogger(); logger.level = 'trace'; let timerID = 0; // Pull feeds from config files: , // feeds.simple_test_feed = require("./config/feed").feed; // Or define in-line feeds.news = require('./feeds/news.json'); feeds.jobsSpecial = require('./feeds/jobs-special.json'); feeds.jobsLocal = require('./feeds/jobs-local.json'); feeds.lifestyle = require('./feeds/lifestyle.json'); feeds.paleo = require('./feeds/paleo.json'); feeds.tech = require('./feeds/tech.json'); feeds.fit = require('./feeds/fit.json'); feeds.bored = require('./feeds/bored.json'); require('dotenv').config({ 'path': `${__dirname }/../.env` }); const braiderOptions = { feeds, 'indent': ' ', 'date_sort_order': 'desc', // Newest first 'log_level': 'warn', 'dedupe_fields': ['link', 'guid'], 'plugins_directories': [`${__dirname }/plugins/`] }; const rssBraider = RssBraider.createClient(braiderOptions); // Override logging level (debug, info, warn, err, off) // rssBraider.logger.level('off'); function gatherV2(feedName, fileName, mode) { return new Promise((resolve, reject) => { const modeSuffix = { 'rss': '.xml', 'json': '.json' }; logger.info(`Gathering ${feedName} as ${modeSuffix[mode]}...`); rssBraider.processFeed(feedName, mode, (err, data) => { if (err) { logger.error(err); return reject(err); } fsWriteFile(`${__dirname}/dist/${fileName}${modeSuffix[mode]}`, data).then(function () { return resolve(`${feedName} saved`); }); }); }); } function gatherV2Email(feedName, fileName, mode) { return new Promise((resolve, reject) => { const modeSuffix = { 'rss': '.xml', 'json': '.json' }; logger.info(`Gathering ${feedName} as ${modeSuffix[mode]}...`); rssBraider.processFeed(feedName, mode, (err, data) => { if (err) { logger.error(err); return reject(err); } fsWriteFile(`${__dirname}/dist/${fileName}${modeSuffix[mode]}`, data).then(function () { return resolve(`${feedName} saved`); }); }); }); } function toHour(extra = 0) { const now = new Date(); return (3600000 - (now.getTime() % 3600000)) + extra; } async function gatherXML(feedName, fileName) { rssBraider.processFeed(feedName, 'rss', function (err, data) { if (err) return console.log(err); console.log('Saving', `${__dirname }/dist/${ fileName }.xml`); fs.writeFile(`${__dirname }/dist/${ fileName }.xml`, data, function (err) { if (err) return console.log(err); console.log('The file was saved!'); }); }); } async function main() { /*await gatherV2('jobsSpecial', 'jobs-special', 'rss').then((d) => { logger.debug(d); }).catch((e) => { logger.error(e); });*/ await gatherV2('news', 'news', 'json').then((d) => { logger.info(d); }).catch((e) => { logger.error(e); }); await gatherV2('lifestyle', 'lifestyle', 'json').then((d) => { logger.info(d); }).catch((e) => { logger.error(e); }); await gatherV2('paleo', 'paleo', 'json').then((d) => { logger.info(d); }).catch((e) => { logger.error(e); }); await gatherV2('tech', 'tech', 'json').then((d) => { logger.info(d); }).catch((e) => { logger.error(e); }); await gatherV2('fit', 'fit', 'json').then((d) => { logger.info(d); }).catch((e) => { logger.error(e); }); await gatherV2('bored', 'bored', 'json').then((d) => { logger.info(d); }).catch((e) => { logger.error(e); }); /* await gatherV2('jobsSpecial', 'jobs-special', 'json').then((d) => { logger.debug(d); }).catch((e) => { logger.error(e); }); await gatherV2('jobsLocal', 'jobs-local', 'rss').then((d) => { logger.debug(d); }).catch((e) => { logger.error(e); });*/ // await gatherXML('jobsLocal','jobs-local'); } function test() { gatherV2Email('jobsSpecial', 'jobs-special', 'json').then((d) => { logger.debug(d); }).catch((e) => { logger.error(e); }); } function tick() { const extra = Math.floor((Math.random() * 60) + 1) * 1000; main(); timerID = setTimeout( () => tick(), toHour(extra) ); } (() => { logger.info('Gather started...'); timerID = setTimeout( () => tick(), toHour() ); main(); })();