martin/rss-braider
1
0
Fork 0
mirror of https://gitlab.silvrtree.co.uk/martind2000/rss-braider.git synced 2025-03-11 07:30:02 +00:00
Code Issues Projects Releases Wiki Activity

Fixing plugin issue that appeared under high load. Adding 'source' to plugins

Browse Source
This commit is contained in:
Kip Gebhardt 2015-03-03 15:58:56 -08:00
parent 4284419908
commit 76772ddded
7 changed files with 26 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
examples/config/feed.js
View File

@ -16,6 +16,11 @@ var feed = {
"name" : "NPR Health",
"count" : 1,
"file_path" : __dirname + "/../feed_xml/npr_health.xml",
},
{
"name" : "NPR Sports",
"count" : 1,
"feed_url" : "http://www.npr.org/rss/rss.php?id=1055"
}
]
};

5
examples/config/feed_with_plugins.js
View File

@ -24,6 +24,11 @@ var feed = {
"name" : "NPR Health",
"count" : 1,
"file_path" : __dirname + "/../feed_xml/npr_health.xml",
},
{
"name" : "NPR Sports",
"count" : 1,
"feed_url" : "http://www.npr.org/rss/rss.php?id=1055"
}
]
};

29
lib/RssBraider.js
View File

@ -53,14 +53,9 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
format = 'rss';
}
var self = this,
feed = this.feeds[feed_name],
feed = self.feeds[feed_name],
feed_articles = [];
// set these for the request
self.feed_name = feed_name;
self.format = format;
self.feed = feed;
// logger.info("DEBUG processFeed: feed is set to " + feed_name);
if (!feed || !feed.sources || feed.sources.length < 1) {
@ -68,11 +63,12 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
}
async.each(feed.sources, function(source, callback) {
var count = source.count || feed.default_count || 10,
var count = source.count || feed.default_count || 10, // Number of articles
url = source.feed_url || null,
file_path = source.file_path || null,
source_articles = [];
logger.debug("Requesting source:" + source.name + " at " + url + " for feed:" + feed_name);
// todo: Check if source.file is set and set up a fs stream read
var feedparser = new FeedParser();
if (url) {
@ -97,6 +93,7 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
filestream.pipe(feedparser);
} else {
logger.error("url or file_path not defined for feed: " + source.name);
return callback();
}
feedparser.on('error', function(error) {
@ -113,7 +110,8 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
if (source.feed_url) {
item.source_url = source.feed_url;
}
var article = self.processItem(item, source);
// Process Item/Article
var article = self.processItem(item, source, feed_name);
if (article) {
source_articles.push(article);
}
@ -155,7 +153,6 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
var newfeed = new RSS(options, feed_articles);
var ret_string;
switch (format.toLowerCase()) {
case 'json':
@ -176,7 +173,7 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
};
// Accepts a feed-parser item and builds a node-rss itemOptions object
RssBraider.prototype.processItem = function (item, source) {
RssBraider.prototype.processItem = function (item, source, feed_name) {
var self = this;
if (!item) {
@ -199,19 +196,17 @@ RssBraider.prototype.processItem = function (item, source) {
// Run the plugins specified by the "plugins" section of the
// feed config file to build out any custom elements or
// do transforms
self.runPlugins(item, itemOptions, source);
self.runPlugins(item, itemOptions, source, feed_name);
// logger.info("returning for item.guid:" + item.guid);
return itemOptions;
};
RssBraider.prototype.runPlugins = function (item, itemOptions, source) {
RssBraider.prototype.runPlugins = function (item, itemOptions, source, feed_name) {
var self = this,
feed = self.feed,
feed_plugins = feed.plugins || [];
feed = self.feeds[feed_name] || {},
plugins_list = feed.plugins || [];
// Process the item through the desired feed plugins
feed_plugins.forEach(function(plugin_name){
plugins_list.forEach(function(plugin_name){
if (self.plugins[plugin_name]) {
// logger.info("DEBUG runPlugins running " + plugin_name + " for item " + item.guid + " in feed: " + feed.meta.title);
self.plugins[plugin_name](item, itemOptions, source);

2
lib/plugins/add_media_thumbnail.js
View File

@ -5,7 +5,7 @@
// else
// 'media:thumbnail'
var _ = require('lodash');
module.exports = function (item, itemOptions) {
module.exports = function (item, itemOptions, source) {
if (!item || !itemOptions) {
return;
}

3
lib/plugins/content_encoded.js
View File

@ -3,7 +3,7 @@
// <content:encoded>
// <![CDATA[<p>Stewart let the news slip during a taping of his show today.]]>
// </content:encoded>
module.exports = function (item, itemOptions) {
module.exports = function (item, itemOptions, source) {
if (!item || !itemOptions) {
return;
}
@ -17,5 +17,4 @@ module.exports = function (item, itemOptions) {
}
);
}
return;
};

2
lib/plugins/strip_elements_from_powerpress.js
View File

@ -1,4 +1,4 @@
module.exports = function (item, itemOptions) {
module.exports = function (item, itemOptions, source) {
if (!item || !itemOptions) {
return;
}

2
lib/plugins/wfw_slash_comments.js
View File

@ -1,4 +1,4 @@
module.exports = function (item, itemOptions) {
module.exports = function (item, itemOptions, source) {
if (!item || !itemOptions) {
return;
}