martin/rss-braider
1
0
Fork 0
mirror of https://gitlab.silvrtree.co.uk/martind2000/rss-braider.git synced 2025-02-11 06:39:16 +00:00
Code Issues Projects Releases Wiki Activity

Cleaning up logging calls.

Browse Source
This commit is contained in:
Kip Gebhardt 2015-06-11 15:48:13 -07:00
parent d3c2e4c6fc
commit ca18144273
2 changed files with 24 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
lib/RssBraider.js
View File

@ -1,5 +1,4 @@
// process feed-reader item into node-rss item // process feed-reader item into node-rss item
var FeedParser = require('feedparser'), var FeedParser = require('feedparser'),
bunyan = require('bunyan'), bunyan = require('bunyan'),
_ = require('lodash'), _ = require('lodash'),
@ -11,16 +10,14 @@ var FeedParser = require('feedparser'),
var logger; var logger;
var RssBraider = function (options) { var RssBraider = function (options) {
this.feeds = options.feeds || null; this.feeds = options.feeds || null;
this.logger = logger = options.logger || bunyan.createLogger({name: 'rss-braider'}); this.logger = options.logger || bunyan.createLogger({name: 'rss-braider'});
this.indent = options.indent || " "; this.indent = options.indent || " ";
this.dedupe_fields = options.dedupe_fields || []; // The fields to use to identify duplicate articles this.dedupe_fields = options.dedupe_fields || []; // The fields to use to identify duplicate articles
this.date_sort_order = options.date_sort_order || "desc"; this.date_sort_order = options.date_sort_order || "desc";
this.plugins_directories = options.plugins_directories || []; this.plugins_directories = options.plugins_directories || [];
// load plugins from plugins folder
// TODO, specify plugins location
this.plugins = {}; this.plugins = {};
this.loadPlugins(); this.loadPlugins();
}; };
// loadup self.plugins with the plugin functions // loadup self.plugins with the plugin functions
@ -28,7 +25,7 @@ RssBraider.prototype.loadPlugins = function () {
var self = this; var self = this;
if (self.plugins_directories.length < 1) { if (self.plugins_directories.length < 1) {
// logger.info("No plugins_directories specified. No plugins loaded."); // self.logger.info("No plugins_directories specified. No plugins loaded.");
} }
self.plugins_directories.forEach(function(path){ self.plugins_directories.forEach(function(path){
// load up each file and assign it to the plugins // load up each file and assign it to the plugins
@ -36,10 +33,10 @@ RssBraider.prototype.loadPlugins = function () {
filenames.forEach(function(filename){ filenames.forEach(function(filename){
var plugin_name = filename.replace(/.js$/, ''); var plugin_name = filename.replace(/.js$/, '');
if (self.plugins[plugin_name]) { if (self.plugins[plugin_name]) {
logger.warn("Duplicate plugin name: ", plugin_name, "Overwriting with newer plugin"); self.logger.warn("Duplicate plugin name: ", plugin_name, "Overwriting with newer plugin");
} }
self.plugins[plugin_name] = require(path + '/' + plugin_name); self.plugins[plugin_name] = require(path + '/' + plugin_name);
// logger.info("plugin registered:", plugin_name); // self.logger.info("plugin registered:", plugin_name);
}); });
}); });
}; };
@ -63,7 +60,7 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
feed = self.feeds[feed_name], feed = self.feeds[feed_name],
feed_articles = []; feed_articles = [];
// logger.info("DEBUG processFeed: feed is set to " + feed_name); // self.logger.info("DEBUG processFeed: feed is set to " + feed_name);
if (!feed || !feed.sources || feed.sources.length < 1) { if (!feed || !feed.sources || feed.sources.length < 1) {
return callback("No definition for feed name: " + feed_name); return callback("No definition for feed name: " + feed_name);
@ -75,16 +72,16 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
file_path = source.file_path || null, file_path = source.file_path || null,
source_articles = []; source_articles = [];
// logger.debug("Requesting source:" + source.name + " at " + url + " for feed:" + feed_name); // self.logger.debug("Requesting source:" + source.name + " at " + url + " for feed:" + feed_name);
// todo: Check if source.file is set and set up a fs stream read // todo: Check if source.file is set and set up a fs stream read
var feedparser = new FeedParser(); var feedparser = new FeedParser();
if (url) { if (url) {
var req = request(url); var req = request(url);
// logger.info("request to", url); // self.logger.info("request to", url);
req.on('error', function (error) { req.on('error', function (error) {
logger.error(error); self.logger.error(error);
}); });
req.on('response', function (res) { req.on('response', function (res) {
@ -99,21 +96,20 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
var filestream = fs.createReadStream(file_path); var filestream = fs.createReadStream(file_path);
filestream.pipe(feedparser); filestream.pipe(feedparser);
} else { } else {
logger.error("url or file_path not defined for feed: " + source.name); self.logger.error("url or file_path not defined for feed: " + source.name);
return callback(); return callback();
} }
feedparser.on('error', function(error) { feedparser.on('error', function(error) {
logger.error("feedparser error:", error, "name:", source.name, "source:", source.feed_url); self.logger.error("feedparser error:", error, "name:", source.name, "source:", source.feed_url);
}); });
// Collect the articles from this source // Collect the articles from this source
feedparser.on('readable', function() { feedparser.on('readable', function() {
// This is where the action is!
var stream = this, var stream = this,
item; item;
while ( item = stream.read() ) { while ( !!(item = stream.read()) ) {
if (source.feed_url) { if (source.feed_url) {
item.source_url = source.feed_url; item.source_url = source.feed_url;
} }
@ -121,6 +117,8 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
var article = self.processItem(item, source, feed_name); var article = self.processItem(item, source, feed_name);
if (article) { if (article) {
source_articles.push(article); source_articles.push(article);
} else {
self.logger.debug("Empty article found in " + feed_name, item.title);
} }
} }
}); });
@ -136,7 +134,7 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
}, },
function(err){ function(err){
if (err) { if (err) {
logger.error(err); self.logger.error(err);
return callback(err); return callback(err);
} else { } else {
// Final Dedupe step and resort // Final Dedupe step and resort
@ -166,11 +164,10 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
ret_string = JSON.stringify(newfeed); ret_string = JSON.stringify(newfeed);
break; break;
case 'rss': case 'rss':
case 'xml':
ret_string = newfeed.xml(self.indent); ret_string = newfeed.xml(self.indent);
break; break;
default: default:
logger.error("Unknown format:", format); self.logger.error("Unknown format:", format);
ret_string = "{}"; ret_string = "{}";
} }
@ -183,8 +180,8 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
RssBraider.prototype.processItem = function (item, source, feed_name) { RssBraider.prototype.processItem = function (item, source, feed_name) {
var self = this; var self = this;
if (!item) { if (!item || !source || !feed_name) {
logger.error("processItem: no item passed in"); self.logger.error("processItem: missing item, source, and/or feed_name");
return null; return null;
} }
// Basics // Basics
@ -201,8 +198,8 @@ RssBraider.prototype.processItem = function (item, source, feed_name) {
}; };
// Run the plugins specified by the "plugins" section of the // Run the plugins specified by the "plugins" section of the
// feed config file to build out any custom elements or // feed .js file to build out any custom elements or
// do transforms // do transforms/filters
itemOptions = self.runPlugins(item, itemOptions, source, feed_name); itemOptions = self.runPlugins(item, itemOptions, source, feed_name);
return itemOptions; return itemOptions;
@ -218,19 +215,15 @@ RssBraider.prototype.runPlugins = function (item, itemOptions, source, feed_name
for (var i = 0; i < plugins_list.length; i++) { for (var i = 0; i < plugins_list.length; i++) {
var plugin_name = plugins_list[i]; var plugin_name = plugins_list[i];
if (self.plugins[plugin_name]) { if (self.plugins[plugin_name]) {
// logger.info("DEBUG runPlugins running " + plugin_name + " for item " + item.guid + " in feed: " + feed.meta.title);
itemOptions = self.plugins[plugin_name](item, itemOptions, source); itemOptions = self.plugins[plugin_name](item, itemOptions, source);
} else { } else {
logger.error("A plugin named '" + plugin_name + "' hasn't been registered"); self.logger.error("A plugin named '" + plugin_name + "' hasn't been registered");
} }
// If itemOptions comes back null, skip this item // If itemOptions comes back null, skip this item as one of the plugins decided to toss it
// as one of the plugins decided to toss it
if (itemOptions === null) { if (itemOptions === null) {
// logger.info("Plugin rejected item", plugin_name, item.guid, feed.meta.title); self.logger.debug("Plugin '" + plugin_name + "' filtered out item: '" + item.title + "' from feed '" + feed.meta.title + "'");
break; break;
} else {
// logger.info("Plugin completed", plugin_name, item.guid, feed.meta.title);
} }
} }
return itemOptions; return itemOptions;

1
test/index.js
View File

@ -122,6 +122,7 @@ test('filter all articles out using plugin', function(t) {
plugins_directories : [__dirname + '/../lib/example_plugins/'] plugins_directories : [__dirname + '/../lib/example_plugins/']
}; };
var rss_braider = RssBraider.createClient(braider_options); var rss_braider = RssBraider.createClient(braider_options);
rss_braider.logger.level('info');
rss_braider.processFeed('sample_feed', 'rss', function(err, data){ rss_braider.processFeed('sample_feed', 'rss', function(err, data){
if (err) { if (err) {