mirror of
https://gitlab.silvrtree.co.uk/martind2000/rss-braider.git
synced 2025-02-10 23:29:15 +00:00
Cleaning up logging calls.
This commit is contained in:
Kip Gebhardt
parent
d3c2e4c6fc
commit
ca18144273
@ -1,5 +1,4 @@
|
||||
// process feed-reader item into node-rss item
|
||||
|
||||
var FeedParser = require('feedparser'),
|
||||
bunyan = require('bunyan'),
|
||||
_ = require('lodash'),
|
||||
@ -11,16 +10,14 @@ var FeedParser = require('feedparser'),
|
||||
var logger;
|
||||
var RssBraider = function (options) {
|
||||
this.feeds = options.feeds || null;
|
||||
this.logger = logger = options.logger || bunyan.createLogger({name: 'rss-braider'});
|
||||
this.logger = options.logger || bunyan.createLogger({name: 'rss-braider'});
|
||||
this.indent = options.indent || " ";
|
||||
this.dedupe_fields = options.dedupe_fields || []; // The fields to use to identify duplicate articles
|
||||
this.date_sort_order = options.date_sort_order || "desc";
|
||||
|
||||
this.plugins_directories = options.plugins_directories || [];
|
||||
// load plugins from plugins folder
|
||||
// TODO, specify plugins location
|
||||
this.plugins = {};
|
||||
this.loadPlugins();
|
||||
|
||||
};
|
||||
|
||||
// loadup self.plugins with the plugin functions
|
||||
@ -28,7 +25,7 @@ RssBraider.prototype.loadPlugins = function () {
|
||||
var self = this;
|
||||
|
||||
if (self.plugins_directories.length < 1) {
|
||||
// logger.info("No plugins_directories specified. No plugins loaded.");
|
||||
// self.logger.info("No plugins_directories specified. No plugins loaded.");
|
||||
}
|
||||
self.plugins_directories.forEach(function(path){
|
||||
// load up each file and assign it to the plugins
|
||||
@ -36,10 +33,10 @@ RssBraider.prototype.loadPlugins = function () {
|
||||
filenames.forEach(function(filename){
|
||||
var plugin_name = filename.replace(/.js$/, '');
|
||||
if (self.plugins[plugin_name]) {
|
||||
logger.warn("Duplicate plugin name: ", plugin_name, "Overwriting with newer plugin");
|
||||
self.logger.warn("Duplicate plugin name: ", plugin_name, "Overwriting with newer plugin");
|
||||
}
|
||||
self.plugins[plugin_name] = require(path + '/' + plugin_name);
|
||||
// logger.info("plugin registered:", plugin_name);
|
||||
// self.logger.info("plugin registered:", plugin_name);
|
||||
});
|
||||
});
|
||||
};
|
||||
@ -63,7 +60,7 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
|
||||
feed = self.feeds[feed_name],
|
||||
feed_articles = [];
|
||||
|
||||
// logger.info("DEBUG processFeed: feed is set to " + feed_name);
|
||||
// self.logger.info("DEBUG processFeed: feed is set to " + feed_name);
|
||||
|
||||
if (!feed || !feed.sources || feed.sources.length < 1) {
|
||||
return callback("No definition for feed name: " + feed_name);
|
||||
@ -75,16 +72,16 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
|
||||
file_path = source.file_path || null,
|
||||
source_articles = [];
|
||||
|
||||
// logger.debug("Requesting source:" + source.name + " at " + url + " for feed:" + feed_name);
|
||||
// self.logger.debug("Requesting source:" + source.name + " at " + url + " for feed:" + feed_name);
|
||||
// todo: Check if source.file is set and set up a fs stream read
|
||||
var feedparser = new FeedParser();
|
||||
if (url) {
|
||||
var req = request(url);
|
||||
|
||||
// logger.info("request to", url);
|
||||
// self.logger.info("request to", url);
|
||||
|
||||
req.on('error', function (error) {
|
||||
logger.error(error);
|
||||
self.logger.error(error);
|
||||
});
|
||||
|
||||
req.on('response', function (res) {
|
||||
@ -99,21 +96,20 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
|
||||
var filestream = fs.createReadStream(file_path);
|
||||
filestream.pipe(feedparser);
|
||||
} else {
|
||||
logger.error("url or file_path not defined for feed: " + source.name);
|
||||
self.logger.error("url or file_path not defined for feed: " + source.name);
|
||||
return callback();
|
||||
}
|
||||
|
||||
feedparser.on('error', function(error) {
|
||||
logger.error("feedparser error:", error, "name:", source.name, "source:", source.feed_url);
|
||||
self.logger.error("feedparser error:", error, "name:", source.name, "source:", source.feed_url);
|
||||
});
|
||||
|
||||
// Collect the articles from this source
|
||||
feedparser.on('readable', function() {
|
||||
// This is where the action is!
|
||||
var stream = this,
|
||||
item;
|
||||
|
||||
while ( item = stream.read() ) {
|
||||
while ( !!(item = stream.read()) ) {
|
||||
if (source.feed_url) {
|
||||
item.source_url = source.feed_url;
|
||||
}
|
||||
@ -121,6 +117,8 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
|
||||
var article = self.processItem(item, source, feed_name);
|
||||
if (article) {
|
||||
source_articles.push(article);
|
||||
} else {
|
||||
self.logger.debug("Empty article found in " + feed_name, item.title);
|
||||
}
|
||||
}
|
||||
});
|
||||
@ -136,7 +134,7 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
|
||||
},
|
||||
function(err){
|
||||
if (err) {
|
||||
logger.error(err);
|
||||
self.logger.error(err);
|
||||
return callback(err);
|
||||
} else {
|
||||
// Final Dedupe step and resort
|
||||
@ -166,11 +164,10 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
|
||||
ret_string = JSON.stringify(newfeed);
|
||||
break;
|
||||
case 'rss':
|
||||
case 'xml':
|
||||
ret_string = newfeed.xml(self.indent);
|
||||
break;
|
||||
default:
|
||||
logger.error("Unknown format:", format);
|
||||
self.logger.error("Unknown format:", format);
|
||||
ret_string = "{}";
|
||||
}
|
||||
|
||||
@ -183,8 +180,8 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
|
||||
RssBraider.prototype.processItem = function (item, source, feed_name) {
|
||||
var self = this;
|
||||
|
||||
if (!item) {
|
||||
logger.error("processItem: no item passed in");
|
||||
if (!item || !source || !feed_name) {
|
||||
self.logger.error("processItem: missing item, source, and/or feed_name");
|
||||
return null;
|
||||
}
|
||||
// Basics
|
||||
@ -201,8 +198,8 @@ RssBraider.prototype.processItem = function (item, source, feed_name) {
|
||||
};
|
||||
|
||||
// Run the plugins specified by the "plugins" section of the
|
||||
// feed config file to build out any custom elements or
|
||||
// do transforms
|
||||
// feed .js file to build out any custom elements or
|
||||
// do transforms/filters
|
||||
itemOptions = self.runPlugins(item, itemOptions, source, feed_name);
|
||||
|
||||
return itemOptions;
|
||||
@ -218,19 +215,15 @@ RssBraider.prototype.runPlugins = function (item, itemOptions, source, feed_name
|
||||
for (var i = 0; i < plugins_list.length; i++) {
|
||||
var plugin_name = plugins_list[i];
|
||||
if (self.plugins[plugin_name]) {
|
||||
// logger.info("DEBUG runPlugins running " + plugin_name + " for item " + item.guid + " in feed: " + feed.meta.title);
|
||||
itemOptions = self.plugins[plugin_name](item, itemOptions, source);
|
||||
} else {
|
||||
logger.error("A plugin named '" + plugin_name + "' hasn't been registered");
|
||||
self.logger.error("A plugin named '" + plugin_name + "' hasn't been registered");
|
||||
}
|
||||
|
||||
// If itemOptions comes back null, skip this item
|
||||
// as one of the plugins decided to toss it
|
||||
// If itemOptions comes back null, skip this item as one of the plugins decided to toss it
|
||||
if (itemOptions === null) {
|
||||
// logger.info("Plugin rejected item", plugin_name, item.guid, feed.meta.title);
|
||||
self.logger.debug("Plugin '" + plugin_name + "' filtered out item: '" + item.title + "' from feed '" + feed.meta.title + "'");
|
||||
break;
|
||||
} else {
|
||||
// logger.info("Plugin completed", plugin_name, item.guid, feed.meta.title);
|
||||
}
|
||||
}
|
||||
return itemOptions;
|
||||
|
||||
@ -122,6 +122,7 @@ test('filter all articles out using plugin', function(t) {
|
||||
plugins_directories : [__dirname + '/../lib/example_plugins/']
|
||||
};
|
||||
var rss_braider = RssBraider.createClient(braider_options);
|
||||
rss_braider.logger.level('info');
|
||||
|
||||
rss_braider.processFeed('sample_feed', 'rss', function(err, data){
|
||||
if (err) {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user