const RssBraider = require('rss-braider'); const fs = require('fs'); const ejs = require( 'ejs'); const read = require('fs').readFileSync; const join = require('path').join; const cron = require('node-cron'); const str = read( join(__dirname, '/templates/rss.ejs'), 'utf8'); const feeds = {}; const log4js = require('log4js'); const logger = log4js.getLogger(); logger.level = 'trace'; // Pull feeds from config files: , // feeds.simple_test_feed = require("./config/feed").feed; // Or define in-line feeds.news = require('./feeds/news.json'); feeds.jobsSpecial = require('./feeds/jobs-special.json'); feeds.jobsLocal = require('./feeds/jobs-local.json'); feeds.lifestyle = require('./feeds/lifestyle.json'); feeds.paleo = require('./feeds/paleo.json'); feeds.tech = require('./feeds/tech.json'); feeds.fit = require('./feeds/fit.json'); feeds.bored = require('./feeds/bored.json'); const braider_options = { feeds: feeds, indent: ' ', date_sort_order: 'desc', // Newest first log_level: 'warn', dedupe_fields: ['link', 'guid'], plugins_directories: [__dirname + '/plugins/'] }; const rss_braider = RssBraider.createClient(braider_options); // Override logging level (debug, info, warn, err, off) //rss_braider.logger.level('off'); function gatherV2(feedName, fileName, mode) { const modeSuffix = {'rss': '.xml', 'json': '.json'}; return new Promise((resolve, reject) => { logger.info(`Gathering ${feedName}...`); rss_braider.processFeed(feedName, mode, function (err, data) { if (err) { logger.error(err); return reject(err); } fs.writeFile(`${__dirname}/dist/${fileName}${modeSuffix[mode]}`, data, function (err) { if (err) { logger.error(err); return reject(err); } return resolve(`${feedName} saved`); }); }); }); } async function gather(feedName, fileName) { logger.info(`Gathering ${feedName}...`); rss_braider.processFeed(feedName, 'json', function (err, data) { console.log('A'); if (err) { logger.error(err); return err; } console.log('B'); const j = JSON.parse(data); console.log('C'); const ejsOutput = ejs.compile(str)(j); console.log('D'); fs.writeFile(__dirname + '/dist/' + fileName + '.html', ejsOutput, function (err) { if (err) { logger.error(err); return console.error(err); } console.log('The file was saved!'); }); fs.writeFile(__dirname + '/dist/' + fileName + '.json', data, function (err) { if (err) { logger.error(err); return console.error(err); } console.log('The file was saved!'); }); }); } async function gatherXML(feedName, fileName) { rss_braider.processFeed(feedName, 'rss', function (err, data) { if (err) { return console.log(err); } console.log('Saving', __dirname + '/dist/' + fileName + '.xml'); fs.writeFile(__dirname + '/dist/' + fileName + '.xml', data, function (err) { if (err) { return console.log(err); } console.log('The file was saved!'); }); }); } async function main() { await gatherV2('jobsSpecial', 'jobs-special', 'rss').then((d) => { logger.info(d); }).catch((e) => { logger.error(e); }); await gatherV2('news', 'news', 'json').then((d) => { logger.info(d); }).catch((e) => { logger.error(e); }); await gatherV2('lifestyle', 'lifestyle', 'json').then((d) => { logger.info(d); }).catch((e) => { logger.error(e); }); await gatherV2('paleo', 'paleo', 'json').then((d) => { logger.info(d); }).catch((e) => { logger.error(e); }); await gatherV2('tech', 'tech', 'json').then((d) => { logger.info(d); }).catch((e) => { logger.error(e); }); await gatherV2('fit', 'fit', 'json').then((d) => { logger.info(d); }).catch((e) => { logger.error(e); }); await gatherV2('bored', 'bored', 'json').then((d) => { logger.info(d); }).catch((e) => { logger.error(e); }); // await gatherXML('jobsLocal','jobs-local'); } cron.schedule('3 * * * *', function () { main(); return -1; }); main();