diff --git a/lib/RssBraider.js b/lib/RssBraider.js index 2e656a6..d6970eb 100644 --- a/lib/RssBraider.js +++ b/lib/RssBraider.js @@ -1,5 +1,4 @@ // process feed-reader item into node-rss item - var FeedParser = require('feedparser'), bunyan = require('bunyan'), _ = require('lodash'), @@ -11,16 +10,14 @@ var FeedParser = require('feedparser'), var logger; var RssBraider = function (options) { this.feeds = options.feeds || null; - this.logger = logger = options.logger || bunyan.createLogger({name: 'rss-braider'}); + this.logger = options.logger || bunyan.createLogger({name: 'rss-braider'}); this.indent = options.indent || " "; this.dedupe_fields = options.dedupe_fields || []; // The fields to use to identify duplicate articles this.date_sort_order = options.date_sort_order || "desc"; + this.plugins_directories = options.plugins_directories || []; - // load plugins from plugins folder - // TODO, specify plugins location this.plugins = {}; this.loadPlugins(); - }; // loadup self.plugins with the plugin functions @@ -28,7 +25,7 @@ RssBraider.prototype.loadPlugins = function () { var self = this; if (self.plugins_directories.length < 1) { - // logger.info("No plugins_directories specified. No plugins loaded."); + // self.logger.info("No plugins_directories specified. No plugins loaded."); } self.plugins_directories.forEach(function(path){ // load up each file and assign it to the plugins @@ -36,10 +33,10 @@ RssBraider.prototype.loadPlugins = function () { filenames.forEach(function(filename){ var plugin_name = filename.replace(/.js$/, ''); if (self.plugins[plugin_name]) { - logger.warn("Duplicate plugin name: ", plugin_name, "Overwriting with newer plugin"); + self.logger.warn("Duplicate plugin name: ", plugin_name, "Overwriting with newer plugin"); } self.plugins[plugin_name] = require(path + '/' + plugin_name); - // logger.info("plugin registered:", plugin_name); + // self.logger.info("plugin registered:", plugin_name); }); }); }; @@ -63,7 +60,7 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback) feed = self.feeds[feed_name], feed_articles = []; - // logger.info("DEBUG processFeed: feed is set to " + feed_name); + // self.logger.info("DEBUG processFeed: feed is set to " + feed_name); if (!feed || !feed.sources || feed.sources.length < 1) { return callback("No definition for feed name: " + feed_name); @@ -75,16 +72,16 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback) file_path = source.file_path || null, source_articles = []; - // logger.debug("Requesting source:" + source.name + " at " + url + " for feed:" + feed_name); + // self.logger.debug("Requesting source:" + source.name + " at " + url + " for feed:" + feed_name); // todo: Check if source.file is set and set up a fs stream read var feedparser = new FeedParser(); if (url) { var req = request(url); - // logger.info("request to", url); + // self.logger.info("request to", url); req.on('error', function (error) { - logger.error(error); + self.logger.error(error); }); req.on('response', function (res) { @@ -99,21 +96,20 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback) var filestream = fs.createReadStream(file_path); filestream.pipe(feedparser); } else { - logger.error("url or file_path not defined for feed: " + source.name); + self.logger.error("url or file_path not defined for feed: " + source.name); return callback(); } feedparser.on('error', function(error) { - logger.error("feedparser error:", error, "name:", source.name, "source:", source.feed_url); + self.logger.error("feedparser error:", error, "name:", source.name, "source:", source.feed_url); }); // Collect the articles from this source feedparser.on('readable', function() { - // This is where the action is! var stream = this, item; - while ( item = stream.read() ) { + while ( !!(item = stream.read()) ) { if (source.feed_url) { item.source_url = source.feed_url; } @@ -121,6 +117,8 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback) var article = self.processItem(item, source, feed_name); if (article) { source_articles.push(article); + } else { + self.logger.debug("Empty article found in " + feed_name, item.title); } } }); @@ -136,7 +134,7 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback) }, function(err){ if (err) { - logger.error(err); + self.logger.error(err); return callback(err); } else { // Final Dedupe step and resort @@ -166,11 +164,10 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback) ret_string = JSON.stringify(newfeed); break; case 'rss': - case 'xml': ret_string = newfeed.xml(self.indent); break; default: - logger.error("Unknown format:", format); + self.logger.error("Unknown format:", format); ret_string = "{}"; } @@ -183,8 +180,8 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback) RssBraider.prototype.processItem = function (item, source, feed_name) { var self = this; - if (!item) { - logger.error("processItem: no item passed in"); + if (!item || !source || !feed_name) { + self.logger.error("processItem: missing item, source, and/or feed_name"); return null; } // Basics @@ -201,8 +198,8 @@ RssBraider.prototype.processItem = function (item, source, feed_name) { }; // Run the plugins specified by the "plugins" section of the - // feed config file to build out any custom elements or - // do transforms + // feed .js file to build out any custom elements or + // do transforms/filters itemOptions = self.runPlugins(item, itemOptions, source, feed_name); return itemOptions; @@ -218,19 +215,15 @@ RssBraider.prototype.runPlugins = function (item, itemOptions, source, feed_name for (var i = 0; i < plugins_list.length; i++) { var plugin_name = plugins_list[i]; if (self.plugins[plugin_name]) { - // logger.info("DEBUG runPlugins running " + plugin_name + " for item " + item.guid + " in feed: " + feed.meta.title); itemOptions = self.plugins[plugin_name](item, itemOptions, source); } else { - logger.error("A plugin named '" + plugin_name + "' hasn't been registered"); + self.logger.error("A plugin named '" + plugin_name + "' hasn't been registered"); } - // If itemOptions comes back null, skip this item - // as one of the plugins decided to toss it + // If itemOptions comes back null, skip this item as one of the plugins decided to toss it if (itemOptions === null) { - // logger.info("Plugin rejected item", plugin_name, item.guid, feed.meta.title); + self.logger.debug("Plugin '" + plugin_name + "' filtered out item: '" + item.title + "' from feed '" + feed.meta.title + "'"); break; - } else { - // logger.info("Plugin completed", plugin_name, item.guid, feed.meta.title); } } return itemOptions; diff --git a/test/index.js b/test/index.js index 8cb0031..b5be5a3 100644 --- a/test/index.js +++ b/test/index.js @@ -122,6 +122,7 @@ test('filter all articles out using plugin', function(t) { plugins_directories : [__dirname + '/../lib/example_plugins/'] }; var rss_braider = RssBraider.createClient(braider_options); + rss_braider.logger.level('info'); rss_braider.processFeed('sample_feed', 'rss', function(err, data){ if (err) {