Rinser/gather.js

const RssBraider = require('rss-braider');
const fs = require('fs');

const util = require('util');
const fsWriteFile = util.promisify(fs.writeFile);

const feeds = {};

const log4js = require('log4js');
const logger = log4js.getLogger();

logger.level = 'trace';
let timerID = 0;

// Pull feeds from config files: ,
//      feeds.simple_test_feed = require("./config/feed").feed;
// Or define in-line

feeds.news = require('./feeds/news.json');
feeds.jobsSpecial = require('./feeds/jobs-special.json');
feeds.jobsLocal = require('./feeds/jobs-local.json');
feeds.lifestyle = require('./feeds/lifestyle.json');
feeds.paleo = require('./feeds/paleo.json');
feeds.tech = require('./feeds/tech.json');
feeds.fit = require('./feeds/fit.json');
feeds.bored = require('./feeds/bored.json');

const braiderOptions = {
  feeds,
  'indent': '    ',
  'date_sort_order': 'desc', // Newest first
  'log_level': 'warn',
  'dedupe_fields': ['link', 'guid'],
  'plugins_directories': [`${__dirname }/plugins/`]
};
const rssBraider = RssBraider.createClient(braiderOptions);

// Override logging level (debug, info, warn, err, off)
// rssBraider.logger.level('off');

function gatherV2(feedName, fileName, mode) {
  return new Promise((resolve, reject) => {
    const modeSuffix = { 'rss': '.xml', 'json': '.json' };
    logger.info(`Gathering ${feedName} as ${modeSuffix[mode]}...`);
    rssBraider.processFeed(feedName, mode, (err, data) => {
      if (err) {
        logger.error(err);

        return reject(err);
      }

      fsWriteFile(`${__dirname}/dist/${fileName}${modeSuffix[mode]}`, data).then(function () {
        return resolve(`${feedName} saved`);
      });
    });
  });
}

function toHour(extra = 0) {
  const now = new Date();

  return (3600000 - (now.getTime() % 3600000)) + extra;
}

async function gatherXML(feedName, fileName) {
  rssBraider.processFeed(feedName, 'rss', function (err, data) {
    if (err)
      return console.log(err);

    console.log('Saving', `${__dirname }/dist/${ fileName }.xml`);
    fs.writeFile(`${__dirname }/dist/${ fileName }.xml`, data, function (err) {
      if (err)
        return console.log(err);

      console.log('The file was saved!');
    });
  });
}

async function main() {
  await gatherV2('jobsSpecial', 'jobs-special', 'rss').then((d) => {
    logger.debug(d);
  }).catch((e) => {
    logger.error(e);
  });
  await gatherV2('news', 'news', 'json').then((d) => {
    logger.info(d);
  }).catch((e) => {
    logger.error(e);
  });
  await gatherV2('lifestyle', 'lifestyle', 'json').then((d) => {
    logger.info(d);
  }).catch((e) => {
    logger.error(e);
  });
  await gatherV2('paleo', 'paleo', 'json').then((d) => {
    logger.info(d);
  }).catch((e) => {
    logger.error(e);
  });
  await gatherV2('tech', 'tech', 'json').then((d) => {
    logger.info(d);
  }).catch((e) => {
    logger.error(e);
  });
  await gatherV2('fit', 'fit', 'json').then((d) => {
    logger.info(d);
  }).catch((e) => {
    logger.error(e);
  });
  await gatherV2('bored', 'bored', 'json').then((d) => {
    logger.info(d);
  }).catch((e) => {
    logger.error(e);
  });

  await gatherV2('jobsSpecial', 'jobs-special', 'json').then((d) => {
    logger.debug(d);
  }).catch((e) => {
    logger.error(e);
  });
  // await gatherXML('jobsLocal','jobs-local');
}

function test() {
  gatherV2('jobsSpecial', 'jobs-special', 'rss').then((d) => {
    logger.debug(d);
  }).catch((e) => {
    logger.error(e);
  });
}
function tick() {
  const extra = Math.floor((Math.random() * 60) + 1) * 1000;
  main();

  timerID = setTimeout(
    () => tick(),
    toHour(extra)
  );
}

(() => {
  logger.info('Gather started...');
  timerID = setTimeout(
    () => tick(),
    toHour()
  );
  // test();
})();