Fixing plugin issue that appeared under high load. Adding 'source' to plugins

examples/config/feed.js

@@ -16,6 +16,11 @@ var feed = {
             "name"              : "NPR Health",
             "count"             : 1,
             "file_path"         : __dirname + "/../feed_xml/npr_health.xml",
+        },
+        {
+            "name"              : "NPR Sports",
+            "count"             : 1,
+            "feed_url"          : "http://www.npr.org/rss/rss.php?id=1055"
         }
     ]
 };

examples/config/feed_with_plugins.js

@@ -24,6 +24,11 @@ var feed = {
             "name"              : "NPR Health",
             "count"             : 1,
             "file_path"         : __dirname + "/../feed_xml/npr_health.xml",
+        },
+        {
+            "name"              : "NPR Sports",
+            "count"             : 1,
+            "feed_url"          : "http://www.npr.org/rss/rss.php?id=1055"
         }
     ]
 };

lib/RssBraider.js

@@ -53,14 +53,9 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
         format = 'rss';
     }
     var self = this,
-        feed = this.feeds[feed_name],
+        feed = self.feeds[feed_name],
         feed_articles = [];
 
-    // set these for the request
-    self.feed_name = feed_name;
-    self.format = format;
-    self.feed = feed;
-
     // logger.info("DEBUG processFeed: feed is set to " + feed_name);
 
     if (!feed || !feed.sources || feed.sources.length < 1) {
@@ -68,11 +63,12 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
     }
 
     async.each(feed.sources, function(source, callback) {
-        var count = source.count || feed.default_count || 10,
+        var count = source.count || feed.default_count || 10, // Number of articles
             url = source.feed_url || null,
             file_path = source.file_path || null,
             source_articles = [];
 
+        logger.debug("Requesting source:" + source.name + " at " + url + " for feed:" + feed_name);
         // todo: Check if source.file is set and set up a fs stream read
         var feedparser = new FeedParser();
         if (url) {
@@ -97,6 +93,7 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
             filestream.pipe(feedparser);
         } else {
             logger.error("url or file_path not defined for feed: " + source.name);
+            return callback();
         }
 
         feedparser.on('error', function(error) {
@@ -113,7 +110,8 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
                 if (source.feed_url) {
                     item.source_url = source.feed_url;
                 }
-                var article = self.processItem(item, source);
+                // Process Item/Article
+                var article = self.processItem(item, source, feed_name);
                 if (article) {
                     source_articles.push(article);
                 }
@@ -155,7 +153,6 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
 
             var newfeed = new RSS(options, feed_articles);
 
-
             var ret_string;
             switch (format.toLowerCase()) {
                 case 'json':
@@ -176,7 +173,7 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
 };
 
 // Accepts a feed-parser item and builds a node-rss itemOptions object
-RssBraider.prototype.processItem = function (item, source) {
+RssBraider.prototype.processItem = function (item, source, feed_name) {
     var self = this;
 
     if (!item) {
@@ -199,19 +196,17 @@ RssBraider.prototype.processItem = function (item, source) {
     // Run the plugins specified by the "plugins" section of the
     // feed config file to build out any custom elements or
     // do transforms
-    self.runPlugins(item, itemOptions, source);
+    self.runPlugins(item, itemOptions, source, feed_name);
 
-    // logger.info("returning for item.guid:" + item.guid);
     return itemOptions;
 };
 
-RssBraider.prototype.runPlugins = function (item, itemOptions, source) {
+RssBraider.prototype.runPlugins = function (item, itemOptions, source, feed_name) {
     var self = this,
-        feed = self.feed,
+        feed = self.feeds[feed_name] || {},
-        feed_plugins = feed.plugins || [];
+        plugins_list = feed.plugins || [];
-
     // Process the item through the desired feed plugins
-    feed_plugins.forEach(function(plugin_name){
+    plugins_list.forEach(function(plugin_name){
         if (self.plugins[plugin_name]) {
             // logger.info("DEBUG runPlugins running " + plugin_name + " for item " + item.guid + " in feed: " + feed.meta.title);
             self.plugins[plugin_name](item, itemOptions, source);

lib/plugins/add_media_thumbnail.js

@@ -5,7 +5,7 @@
 //       else
 //      'media:thumbnail'
 var _ = require('lodash');
-module.exports = function (item, itemOptions) {
+module.exports = function (item, itemOptions, source) {
     if (!item || !itemOptions) {
         return;
     }

lib/plugins/content_encoded.js

@@ -3,7 +3,7 @@
 //  <content:encoded>
 //      <![CDATA[<p>Stewart let the news slip during a taping of his show today.]]>
 //  </content:encoded>
-module.exports = function (item, itemOptions) {
+module.exports = function (item, itemOptions, source) {
     if (!item || !itemOptions) {
         return;
     }
@@ -17,5 +17,4 @@ module.exports = function (item, itemOptions) {
             }
         );
     }
-    return;
 };

lib/plugins/strip_elements_from_powerpress.js

@@ -1,4 +1,4 @@
-module.exports = function (item, itemOptions) {
+module.exports = function (item, itemOptions, source) {
     if (!item || !itemOptions) {
         return;
     }

lib/plugins/wfw_slash_comments.js

@@ -1,4 +1,4 @@
-module.exports = function (item, itemOptions) {
+module.exports = function (item, itemOptions, source) {
     if (!item || !itemOptions) {
         return;
     }