Rinser/gather.js

const RssBraider = require('rss-braider');
const fs = require('fs');
const ejs = require(
  'ejs');
const read = require('fs').readFileSync;
const join = require('path').join;
const cron = require('node-cron');

const str = read(
  join(__dirname, '/templates/rss.ejs'),
  'utf8');
const feeds = {};

const log4js = require('log4js');
const logger = log4js.getLogger();

logger.level = 'trace';

// Pull feeds from config files: ,
//      feeds.simple_test_feed = require("./config/feed").feed;
// Or define in-line

feeds.news = require('./feeds/news.json');
feeds.jobsSpecial = require('./feeds/jobs-special.json');
feeds.jobsLocal = require('./feeds/jobs-local.json');
feeds.lifestyle = require('./feeds/lifestyle.json');
feeds.paleo = require('./feeds/paleo.json');
feeds.tech = require('./feeds/tech.json');
feeds.fit = require('./feeds/fit.json');
feeds.bored = require('./feeds/bored.json');

const braider_options = {
  feeds: feeds,
  indent: '    ',
  date_sort_order: 'desc', // Newest first
  log_level: 'warn',
  dedupe_fields: ['link', 'guid'],
  plugins_directories: [__dirname + '/plugins/']
};
const rss_braider = RssBraider.createClient(braider_options);

// Override logging level (debug, info, warn, err, off)
//rss_braider.logger.level('off');

function gatherV2(feedName, fileName, mode) {
  const modeSuffix = {'rss': '.xml', 'json': '.json'};
  return new Promise((resolve, reject) => {
    logger.info(`Gathering ${feedName}...`);
    rss_braider.processFeed(feedName, mode, function (err, data) {
      if (err) {
        logger.error(err);
        return reject(err);
      }

      fs.writeFile(`${__dirname}/dist/${fileName}${modeSuffix[mode]}`,
        data,
        function (err) {
          if (err) {
            logger.error(err);
            return reject(err);
          }

          return resolve(`${feedName} saved`);
        });
    });
  });
}


async function gather(feedName, fileName) {
  logger.info(`Gathering ${feedName}...`);
  rss_braider.processFeed(feedName, 'json', function (err, data) {
    console.log('A');
    if (err) {
      logger.error(err);
      return err;
    }

    console.log('B');

    const j = JSON.parse(data);

    console.log('C');
    const ejsOutput = ejs.compile(str)(j);

    console.log('D');
    fs.writeFile(__dirname + '/dist/' + fileName + '.html',
      ejsOutput,
      function (err) {

        if (err) {
          logger.error(err);
          return console.error(err);
        }

        console.log('The file was saved!');
      });

    fs.writeFile(__dirname + '/dist/' + fileName + '.json',
      data,
      function (err) {
        if (err) {
          logger.error(err);
          return console.error(err);
        }

        console.log('The file was saved!');
      });
  });

}

async function gatherXML(feedName, fileName) {
  rss_braider.processFeed(feedName, 'rss', function (err, data) {
    if (err) {
      return console.log(err);
    }
    console.log('Saving', __dirname + '/dist/' + fileName + '.xml');
    fs.writeFile(__dirname + '/dist/' + fileName + '.xml', data, function (err) {
      if (err) {
        return console.log(err);
      }

      console.log('The file was saved!');
    });
  });
}


async function main() {
  await gatherV2('jobsSpecial', 'jobs-special', 'rss').then((d) => {
    logger.info(d);
  }).catch((e) => {
    logger.error(e);
  });
  await gatherV2('news', 'news', 'json').then((d) => {
    logger.info(d);
  }).catch((e) => {
    logger.error(e);
  });
  await gatherV2('lifestyle', 'lifestyle', 'json').then((d) => {
    logger.info(d);
  }).catch((e) => {
    logger.error(e);
  });
  await gatherV2('paleo', 'paleo', 'json').then((d) => {
    logger.info(d);
  }).catch((e) => {
    logger.error(e);
  });
  await gatherV2('tech', 'tech', 'json').then((d) => {
    logger.info(d);
  }).catch((e) => {
    logger.error(e);
  });
  await gatherV2('fit', 'fit', 'json').then((d) => {
    logger.info(d);
  }).catch((e) => {
    logger.error(e);
  });
  await gatherV2('bored', 'bored', 'json').then((d) => {
    logger.info(d);
  }).catch((e) => {
    logger.error(e);
  });
  // await gatherXML('jobsLocal','jobs-local');
}


cron.schedule('3 * * * *', function () {
  main();
  return -1;
});

main();