rss-braider/lib/RssBraider.js

// process feed-reader item into node-rss item

var FeedParser  = require('feedparser'),
    bunyan      = require('bunyan'),
    _           = require('lodash'),
    async       = require('async'),
    request     = require('request'),
    RSS         = require('rss'),
    fs          = require('fs');


var logger;
var RssBraider = function (options) {
    this.feeds = options.feeds || null; // TOOD validate feed configs
    this.logger = logger = options.logger || bunyan.createLogger({name: 'rss-braider'});
    this.indent = options.indent || "    ";
};

RssBraider.prototype.init = function() {
    // Validate the feeds?
};

RssBraider.prototype.feedExists = function (feed_name) {
    if (this.feeds && this.feeds[feed_name]) {
        return true;
    } else {
        return false;
    }
};

RssBraider.prototype.processFeed = function(feed_name, format, callback)
{
    // DEBUG
    console.time("process");

    if (!format) {
        format = 'json';
    }
    var self = this,
        feed = this.feeds[feed_name],
        feed_articles = [];

    if (!feed || !feed.sources || feed.sources.length < 1) {
        return callback("No definition for feed name: " + feed_name);
    }

    async.each(feed.sources, function(source, callback) {
        var count = source.count || feed.default_count || 10,
            url = source.url || null,
            file_path = source.file_path || null,
            source_articles = [];

        // todo: Check if source.file is set and set up a fs stream read
        var feedparser = new FeedParser();
        if (url) {
            var req = request(url);

            logger.info("request to", url);

            req.on('error', function (error) {
                logger.error(error);
            });

            req.on('response', function (res) {
                var stream = this;
                if (res.statusCode !== 200) {
                    return this.emit('error', new Error('Bad status code'));
                }
                stream.pipe(feedparser);
            });
        } else if (file_path) {
            // open file
            var filestream = fs.createReadStream(file_path);
            filestream.pipe(feedparser);
        } else {
            logger.error("url or file_path not defined for feed: " + source.name);
        }

        feedparser.on('error', function(error) {
            logger.error("feedparser: error", error);
        });

        // Collect the articles from this source
        feedparser.on('readable', function() {
            // This is where the action is!
            var stream = this,
                item;

            while ( item = stream.read() ) {
                // logger.info("item received", item.guid);
                var article = self.processItem(item);
                if (article) {
                    source_articles.push(article);
                }
            }
        });

        feedparser.on("end", function(){
            // sort and de-dupe this feed's articles and push them into array
            source_articles = self.dedupe(source_articles);
            source_articles = self.date_sort(source_articles);
            source_articles = source_articles.slice(0, count);
            feed_articles = feed_articles.concat(source_articles);
            callback();
        });
    },
    function(err){
        if (err) {
            logger.error(err);
            return callback(err);
        } else {
            // Sort the stories for the source by date descending
            feed_articles = self.dedupe(feed_articles);
            feed_articles = self.date_sort(feed_articles);
            feed_articles.reverse();

            // Create new feed with these articles
            var options = {
                title               : feed.meta.title,
                site_url            : "http://www.kqed.org",
                description         : feed.meta.description,
                generator           : feed.meta.generator || 'rss-braider',
                feed_url            : feed.meta.url,
                custom_namespaces   : feed.custom_namespaces || [],
                no_cdata_fields     : feed.no_cdata_fields
            };

            var newfeed = new RSS(options, feed_articles);


            var ret_string;
            switch (format.toLowerCase()) {
                case 'json':
                    ret_string = JSON.stringify(newfeed);
                    break;
                case 'rss':
                case 'xml':
                    ret_string = newfeed.xml(self.indent);
                    break;
                default:
                    logger.error("Unknown format:", format);
                    ret_string = "{}";
            }

            // DEBUG
            console.timeEnd("process");
            return callback(null, ret_string);
        }
    });
};

// Accepts a feed-parser item and builds a node-rss itemOptions object
RssBraider.prototype.processItem = function (item) {
    if (!item) {
        logger.error("processItem: no item passed in");
        return null;
    }
    // Basics
    var itemOptions = {
        title           : item.title,
        description     : item.summary,
        url             : item.link,
        guid            : item.guid,
        permalink       : item.permalink,
        author          : item.author,
        date            : item.date,
        custom_elements : []
    };


    //////////////////
    // Custom elements    PLUGINS
    //////////////////

    // TODO: Generify the following if possible

    // content:encoded (i.e. description)
    if (item["content:encoded"] && item["content:encoded"]["#"]){
        var content_encoded = item["content:encoded"]["#"];
        itemOptions.custom_elements.push(
            { "content:encoded":
                {
                    _cdata: content_encoded
                }
            }
        );
    }

    // // wfw
    if (item["wfw:commentrss"] && item["wfw:commentrss"]["#"]){
        itemOptions.custom_elements.push({ "wfw:commentRss": item["wfw:commentrss"]["#"]});
    }

    // // // slash comments
    if (item["slash:comments"] && item["slash:comments"]["#"]){
        itemOptions.custom_elements.push({ "slash:comments": item["slash:comments"]["#"]});
    }

    // Images
    // Take 'media:thumbnail',
    //       else
    //      'media:content'[0]'media:thumbnail'
    //       else
    //      'media:thumbnail'
    if (item['media:thumbnail'] && item['media:thumbnail']['#']) {
        itemOptions.custom_elements.push({'media:thumbnail' : item['media:thumbnail']['#']});
    } else {
        if (item["media:content"]) {
            var media_contents;
            if (! _.isArray(item['media:content'])) {
                media_contents = [item['media:content']];
            } else {
                media_contents = item['media:content'];
            }

            if ( media_contents[0] &&
                 media_contents[0]['media:thumbnail'] &&
                 media_contents[0]['media:thumbnail']['@'] &&
                 media_contents[0]['media:thumbnail']['@'].url) {

                itemOptions.custom_elements.push({'media:thumbnail' : media_contents[0]['media:thumbnail']['@'].url});

            } else {
                itemOptions.custom_elements.push({'media:thumbnail' : media_contents[0]['@'].url});
            }
        }
    }

    return itemOptions;
};

// Dedupe articles in node-rss itemOptions format
RssBraider.prototype.dedupe = function(articles_arr){
    // TODO: sort by guid, url, etc
    return _.uniq(articles_arr);
};

RssBraider.prototype.date_sort = function(articles_arr) {
    var sorted_articles = _.sortBy(articles_arr, function(article) {
        return article.date.getTime();
    });
    return sorted_articles;
};

module.exports = RssBraider;