diff --git a/README.md b/README.md index 61124f0..2f5c04c 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ $ cd examples $ node simple.js (combines 3 sources) $ node plugins.js (combines 3 sources and runs a transformation plugin) ``` -## Code Example +### Code Example ```js var RssBraider = require('rss-braider'), feeds = {}; @@ -48,10 +48,14 @@ feeds.simple_test_feed = { var braider_options = { feeds : feeds, indent : " ", - date_sort_order : "desc" // Newest first + date_sort_order : "desc", // Newest first + log_level : "debug" }; var rss_braider = RssBraider.createClient(braider_options); +// Override logging level (debug, info, warn, err, off) +rss_braider.logger.level('off'); + // Output braided feed as rss. use 'json' for JSON output. rss_braider.processFeed('simple_test_feed', 'rss', function(err, data){ if (err) { @@ -59,4 +63,40 @@ rss_braider.processFeed('simple_test_feed', 'rss', function(err, data){ } console.log(data); }); -``` \ No newline at end of file +``` +## Plugins +Plugins provide custom manipulation and filtering of RSS items/articles. See `examples/plugins` for examples. + +A plugin operates by modifying the `itemOptions` object or by returning `null` which will exclude the `item` (article) from the resulting feed (See `examples/plugins/filter_out_all_articles.js`). + +The `itemsOptions` object gets passed to `node-rss` to generate the RSS feeds, so read the documentation on that module and its use of custom namespaces. (https://github.com/dylang/node-rss) + +### Plugin Example +This plugin will capitalize the article title for all articles +```js +module.exports = function (item, itemOptions, source) { + if (!item || !itemOptions) { + return; + } + + if (itemOptions.title) { + itemOptions.title = itemOptions.title.toUpperCase(); + } + + return itemOptions; +}; +``` + +The plugin is registered with the feed in the feed config .js file and are run in order. +```js +var feed = { + "feed_name" : "feed with plugins", + "default_count" : 1, + "plugins" : ['capitalize_title', 'plugin_template'], +... +``` + +## Release Notes +### 1.0.0 +Changed plugin architecture to allow filtering out of article/items by returning `-1` instead of a modified `itemsOptions` object. This is a breaking change as it will require existing plugins to return `itemsOptions` instead of modifying the reference. See `examples/plugins`. + diff --git a/examples/config/feed_with_plugins.js b/examples/config/feed_with_plugins.js index 76a1563..f31ad2d 100644 --- a/examples/config/feed_with_plugins.js +++ b/examples/config/feed_with_plugins.js @@ -2,7 +2,7 @@ var feed = { "feed_name" : "feed with plugins", "default_count" : 1, "no_cdata_fields" : [], - "plugins" : ['content_encoded'], + "plugins" : ['capitalize_title', 'plugin_template'], "meta" : { "title": "NPR Braided Feed", "description": "This is a test of two NPR sources from file. Plugins are applied." diff --git a/examples/plugins/content_encoded.js b/examples/plugins/add_content_encoded_block.js similarity index 90% rename from examples/plugins/content_encoded.js rename to examples/plugins/add_content_encoded_block.js index d9b477f..f864f49 100644 --- a/examples/plugins/content_encoded.js +++ b/examples/plugins/add_content_encoded_block.js @@ -4,9 +4,6 @@ // Stewart let the news slip during a taping of his show today.]]> // module.exports = function (item, itemOptions, source) { - if (!item || !itemOptions) { - return; - } if (item["content:encoded"] && item["content:encoded"]["#"]){ var content_encoded = item["content:encoded"]["#"]; itemOptions.custom_elements.push( @@ -17,4 +14,5 @@ module.exports = function (item, itemOptions, source) { } ); } + return itemOptions; }; \ No newline at end of file diff --git a/lib/example_plugins/strip_elements_from_powerpress.js b/examples/plugins/add_itunes_elements.js similarity index 55% rename from lib/example_plugins/strip_elements_from_powerpress.js rename to examples/plugins/add_itunes_elements.js index 00931da..e91db79 100644 --- a/lib/example_plugins/strip_elements_from_powerpress.js +++ b/examples/plugins/add_itunes_elements.js @@ -1,27 +1,5 @@ -module.exports = function (item, itemOptions, source) { - if (!item || !itemOptions) { - return; - } - - // 'itunes:summary': - // { '@': {}, - // '#': 'Let KQED Arts help you find the best things to see, do, and explore in San Francisco, Oakland, San Jose, and the surrounding areas.' }, - // 'itunes:author': { '@': {}, '#': 'KQED Arts' }, - // 'itunes:explicit': { '@': {}, '#': 'no' }, - // 'itunes:image': { '@': [Object] }, - // 'itunes:owner': { '@': {}, 'itunes:name': [Object], 'itunes:email': [Object] }, - // 'rss:managingeditor': - // { '@': {}, - // '#': 'ondemand@kqed.org (KQED Arts)', - // name: 'KQED Arts', - // email: 'ondemand@kqed.org' }, - // 'rss:copyright': - // { '@': {}, - // '#': 'Copyright © 2015 KQED Inc. All Rights Reserved.' }, - // 'itunes:subtitle': { '@': {}, '#': 'KQED Public Media for Northern CA' }, - // 'rss:image': { '@': {}, title: [Object], url: [Object], link: [Object] }, - // 'itunes:category': [ [Object], [Object], [Object] ], - +// Pass through itunes content... +// // // Let KQED Arts help you find the best things to see, do, and explore in San Francisco, Oakland, San Jose, and the surrounding areas. // @@ -33,28 +11,9 @@ module.exports = function (item, itemOptions, source) { // ondemand@kqed.org // // ondemand@kqed.org (KQED Arts) -// Copyright © 2015 KQED Inc. All Rights Reserved. // KQED Public Media for Northern CA -// *link -// *comments -// pubDate -// *dc:creator -// *category -// guid -// description -// *content:encoded -// *wfw:commentRss -// *slash:comments -// enclosure -// itunes:subtitle -// itunes:summary -// itunes:author -// itunes:explicit -// *media:content (multiple) -// *media:thumbnail - - // Pass through itunes content +module.exports = function (item, itemOptions, source) { var pass_through_arr = ['itunes:summary', 'itunes:author', 'itunes:explicit', ]; pass_through_arr.forEach(function(element){ if (item[element] && item[element]['#']) { @@ -82,4 +41,5 @@ module.exports = function (item, itemOptions, source) { } }); } + return itemOptions; }; \ No newline at end of file diff --git a/lib/example_plugins/add_media_thumbnail.js b/examples/plugins/add_media_thumbnail.js similarity index 96% rename from lib/example_plugins/add_media_thumbnail.js rename to examples/plugins/add_media_thumbnail.js index 8a482e7..3fe2184 100644 --- a/lib/example_plugins/add_media_thumbnail.js +++ b/examples/plugins/add_media_thumbnail.js @@ -6,10 +6,6 @@ // 'media:thumbnail' var _ = require('lodash'); module.exports = function (item, itemOptions, source) { - if (!item || !itemOptions) { - return; - } - var thumbnail; if (item['media:thumbnail'] && item['media:thumbnail']['#']) { thumbnail = { @@ -51,4 +47,5 @@ module.exports = function (item, itemOptions, source) { } } } + return itemOptions; }; \ No newline at end of file diff --git a/examples/plugins/bad_plugin.js b/examples/plugins/bad_plugin.js new file mode 100644 index 0000000..5588813 --- /dev/null +++ b/examples/plugins/bad_plugin.js @@ -0,0 +1,8 @@ +module.exports = function (item, itemOptions, source) { + + // This is an intentionally broken plugin for testing. + // This doesn't do anything and shouldn't be a template + // for other plugins + + return; +}; \ No newline at end of file diff --git a/examples/plugins/capitalize_title.js b/examples/plugins/capitalize_title.js new file mode 100644 index 0000000..3801a48 --- /dev/null +++ b/examples/plugins/capitalize_title.js @@ -0,0 +1,7 @@ +module.exports = function (item, itemOptions, source) { + if (itemOptions.title) { + itemOptions.title = itemOptions.title.toUpperCase(); + } + + return itemOptions; +}; \ No newline at end of file diff --git a/examples/plugins/filter_out_all_articles.js b/examples/plugins/filter_out_all_articles.js new file mode 100644 index 0000000..82b7da4 --- /dev/null +++ b/examples/plugins/filter_out_all_articles.js @@ -0,0 +1,8 @@ +module.exports = function (item, itemOptions, source) { + if (!item || !itemOptions) { + return; + } + + // This plugin removes all items by returning -1 instead of the processed itemOptions + return -1; +}; \ No newline at end of file diff --git a/lib/example_plugins/kqed.js b/examples/plugins/kqed.js similarity index 94% rename from lib/example_plugins/kqed.js rename to examples/plugins/kqed.js index 39847ed..5271e39 100644 --- a/lib/example_plugins/kqed.js +++ b/examples/plugins/kqed.js @@ -1,8 +1,5 @@ // define kqed source module.exports = function (item, itemOptions, source) { - if (!item || !itemOptions || !source) { - return; - } // Look for kqed namespace elements in source and add as custom elements for item // Ex: // The California Report @@ -36,4 +33,5 @@ module.exports = function (item, itemOptions, source) { { 'kqed:feed_url': item.feed_url } ); } + return itemOptions; }; \ No newline at end of file diff --git a/examples/plugins/plugin_template.js b/examples/plugins/plugin_template.js new file mode 100644 index 0000000..cb30b43 --- /dev/null +++ b/examples/plugins/plugin_template.js @@ -0,0 +1,6 @@ +module.exports = function (item, itemOptions, source) { + // This plugin does no processing + // It's just a template + + return itemOptions; +}; \ No newline at end of file diff --git a/lib/example_plugins/wfw_slash_comments.js b/examples/plugins/wfw_slash_comments.js similarity index 88% rename from lib/example_plugins/wfw_slash_comments.js rename to examples/plugins/wfw_slash_comments.js index c8a96e9..9eef2cf 100644 --- a/lib/example_plugins/wfw_slash_comments.js +++ b/examples/plugins/wfw_slash_comments.js @@ -1,7 +1,4 @@ module.exports = function (item, itemOptions, source) { - if (!item || !itemOptions) { - return; - } // wfw if (item["wfw:commentrss"] && item["wfw:commentrss"]["#"]){ itemOptions.custom_elements.push({ "wfw:commentRss": item["wfw:commentrss"]["#"]}); @@ -11,4 +8,5 @@ module.exports = function (item, itemOptions, source) { if (item["slash:comments"] && item["slash:comments"]["#"]){ itemOptions.custom_elements.push({ "slash:comments": item["slash:comments"]["#"]}); } + return itemOptions; }; \ No newline at end of file diff --git a/examples/simple.js b/examples/simple.js index 5d3101c..d79713d 100644 --- a/examples/simple.js +++ b/examples/simple.js @@ -28,10 +28,14 @@ feeds.simple_test_feed = { var braider_options = { feeds : feeds, indent : " ", - date_sort_order : "desc" // Newest first + date_sort_order : "desc", // Newest first + log_level : 'debug' }; var rss_braider = RssBraider.createClient(braider_options); +// Set logging level (debug, info, warn, err, off) +rss_braider.logger.level('off'); + rss_braider.processFeed('simple_test_feed', 'rss', function(err, data){ if (err) { return console.log(err); diff --git a/examples/plugins.js b/examples/use_plugins.js similarity index 82% rename from examples/plugins.js rename to examples/use_plugins.js index 50ec0fa..983f540 100644 --- a/examples/plugins.js +++ b/examples/use_plugins.js @@ -6,7 +6,8 @@ feed_obj.filefeed = require("./config/feed_with_plugins").feed; var braider_options = { feeds : feed_obj, indent : " ", - plugins_directories : [__dirname + "/plugins/"] + plugins_directories : [__dirname + "/plugins/"], + log_level : 'debug' }; var rss_braider = RssBraider.createClient(braider_options); diff --git a/lib/RssBraider.js b/lib/RssBraider.js index 2937669..eb8c2c6 100644 --- a/lib/RssBraider.js +++ b/lib/RssBraider.js @@ -1,26 +1,32 @@ // process feed-reader item into node-rss item +var FeedParser = require('feedparser'), + bunyan = require('bunyan'), + _ = require('lodash'), + async = require('async'), + request = require('request'), + RSS = require('rss'), + fs = require('fs'), + package_json = require('../package.json'), + logger; -var FeedParser = require('feedparser'), - bunyan = require('bunyan'), - _ = require('lodash'), - async = require('async'), - request = require('request'), - RSS = require('rss'), - fs = require('fs'); - -var logger; var RssBraider = function (options) { + if (!options) { + options = {}; + } this.feeds = options.feeds || null; - this.logger = logger = options.logger || bunyan.createLogger({name: 'rss-braider'}); + this.logger = options.logger || bunyan.createLogger({name: package_json.name}); + + if (options.log_level) { + this.logger.level(options.log_level); + } + this.indent = options.indent || " "; this.dedupe_fields = options.dedupe_fields || []; // The fields to use to identify duplicate articles this.date_sort_order = options.date_sort_order || "desc"; + this.plugins_directories = options.plugins_directories || []; - // load plugins from plugins folder - // TODO, specify plugins location this.plugins = {}; this.loadPlugins(); - }; // loadup self.plugins with the plugin functions @@ -28,7 +34,7 @@ RssBraider.prototype.loadPlugins = function () { var self = this; if (self.plugins_directories.length < 1) { - // logger.info("No plugins_directories specified. No plugins loaded."); + self.logger.debug("No plugins_directories specified. No plugins loaded."); } self.plugins_directories.forEach(function(path){ // load up each file and assign it to the plugins @@ -36,10 +42,10 @@ RssBraider.prototype.loadPlugins = function () { filenames.forEach(function(filename){ var plugin_name = filename.replace(/.js$/, ''); if (self.plugins[plugin_name]) { - logger.warn("Duplicate plugin name: ", plugin_name, "Overwriting with newer plugin"); + self.logger.warn("Duplicate plugin name: ", plugin_name, "Overwriting with newer plugin"); } self.plugins[plugin_name] = require(path + '/' + plugin_name); - // logger.info("plugin registered:", plugin_name); + self.logger.debug("plugin registered:", plugin_name); }); }); }; @@ -56,35 +62,32 @@ RssBraider.prototype.feedExists = function (feed_name) { // trim down to desired count, dedupe and sort RssBraider.prototype.processFeed = function(feed_name, format, callback) { - if (!format) { - format = 'rss'; - } var self = this, feed = self.feeds[feed_name], feed_articles = []; - // logger.info("DEBUG processFeed: feed is set to " + feed_name); + if (!format) { + format = 'rss'; + } if (!feed || !feed.sources || feed.sources.length < 1) { return callback("No definition for feed name: " + feed_name); } + // Process each feed source through Feedparser to get articles. + // Then process each item/article through rss-braider and any plugins async.each(feed.sources, function(source, callback) { - var count = source.count || feed.default_count || 10, // Number of articles + var count = source.count || feed.default_count || 10, // Number of articles per source url = source.feed_url || null, file_path = source.file_path || null, source_articles = []; - logger.debug("Requesting source:" + source.name + " at " + url + " for feed:" + feed_name); - // todo: Check if source.file is set and set up a fs stream read var feedparser = new FeedParser(); if (url) { var req = request(url); - // logger.info("request to", url); - req.on('error', function (error) { - logger.error(error); + self.logger.error(error); }); req.on('response', function (res) { @@ -99,26 +102,27 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback) var filestream = fs.createReadStream(file_path); filestream.pipe(feedparser); } else { - logger.error("url or file_path not defined for feed: " + source.name); + self.logger.error("url or file_path not defined for feed: " + source.name); return callback(); } feedparser.on('error', function(error) { - logger.error("feedparser error:", error, "name:", source.name, "source:", source.feed_url); + self.logger.error("feedparser",", source.name:", source.name, ", url:", source.feed_url, error.stack); }); // Collect the articles from this source feedparser.on('readable', function() { - // This is where the action is! var stream = this, item; - while ( item = stream.read() ) { + while ( !!(item = stream.read()) ) { if (source.feed_url) { item.source_url = source.feed_url; } // Process Item/Article var article = self.processItem(item, source, feed_name); + + // plugins may filter items and return null if (article) { source_articles.push(article); } @@ -126,7 +130,7 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback) }); feedparser.on("end", function(){ - // sort and de-dupe this feed's articles and push them into array + // de-dupe , date sort, and trim this feed's articles and push them into array source_articles = self.dedupe(source_articles, self.dedupe_fields); source_articles = self.date_sort(source_articles); source_articles = source_articles.slice(0, count); @@ -136,14 +140,14 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback) }, function(err){ if (err) { - logger.error(err); + self.logger.error(err); return callback(err); } else { // Final Dedupe step and resort feed_articles = self.dedupe(feed_articles, self.dedupe_fields); feed_articles = self.date_sort(feed_articles); - // Create new feed with these articles + // Create new feed with these articles. Follows node-rss spec var options = { title : feed.meta.title, description : feed.meta.description, @@ -166,11 +170,10 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback) ret_string = JSON.stringify(newfeed); break; case 'rss': - case 'xml': ret_string = newfeed.xml(self.indent); break; default: - logger.error("Unknown format:", format); + self.logger.error("Unknown format:", format); ret_string = "{}"; } @@ -183,8 +186,8 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback) RssBraider.prototype.processItem = function (item, source, feed_name) { var self = this; - if (!item) { - logger.error("processItem: no item passed in"); + if (!item || !source || !feed_name) { + self.logger.error("processItem: missing item, source, and/or feed_name"); return null; } // Basics @@ -201,26 +204,46 @@ RssBraider.prototype.processItem = function (item, source, feed_name) { }; // Run the plugins specified by the "plugins" section of the - // feed config file to build out any custom elements or - // do transforms - self.runPlugins(item, itemOptions, source, feed_name); + // feed .js file to build out any custom elements or + // do transforms/filters + var filteredItemOptions = self.runPlugins(item, itemOptions, source, feed_name); - return itemOptions; + return filteredItemOptions; }; RssBraider.prototype.runPlugins = function (item, itemOptions, source, feed_name) { var self = this, feed = self.feeds[feed_name] || {}, - plugins_list = feed.plugins || []; + plugins_list = feed.plugins || [], + ret_val, + filteredItemOptions; + // Process the item through the desired feed plugins - plugins_list.forEach(function(plugin_name){ + // plugins_list.forEach(function(plugin_name){ + for (var i = 0; i < plugins_list.length; i++) { + var plugin_name = plugins_list[i]; if (self.plugins[plugin_name]) { - // logger.info("DEBUG runPlugins running " + plugin_name + " for item " + item.guid + " in feed: " + feed.meta.title); - self.plugins[plugin_name](item, itemOptions, source); + filteredItemOptions = self.plugins[plugin_name](item, itemOptions, source); } else { - logger.error("A plugin named '" + plugin_name + "' hasn't been registered"); + self.logger.error("A plugin named '" + plugin_name + "' hasn't been registered"); } - }); + + // A plugin returning -1 means skip this item + if (filteredItemOptions === -1) { + self.logger.debug("Plugin '" + plugin_name + "' filtered item from feed '" + feed.meta.title + "'", item.guid); + itemOptions = null; + break; + } + + // Check that the plugin didn't just return null or undef, which would be bad. + if (!filteredItemOptions) { + self.logger.debug("Plugin '" + plugin_name + "' failed to return itemOptions for feed:'" + feed.meta.title + "'", item.guid); + filteredItemOptions = itemOptions; // Reset + } + // Prepare for next plugin. + itemOptions = filteredItemOptions; + } + return itemOptions; }; // Dedupe articles in node-rss itemOptions format @@ -228,6 +251,7 @@ RssBraider.prototype.runPlugins = function (item, itemOptions, source, feed_name // operation on the articles array // TODO, make this a plugin? RssBraider.prototype.dedupe = function(articles_arr, fields){ + var self = this; if ( !fields || fields.length < 1 ) { return _.uniq(articles_arr); } else { @@ -249,8 +273,9 @@ RssBraider.prototype.dedupe = function(articles_arr, fields){ // it's unique deduped_articles.push(article); } else { - // The article matched all of another article's fields - // Do nothing + // The article matched all of another article's "dedupe" fields + // so filter it out (i.e. do nothing) + self.logger.debug("skipping duplicate", '"' + article.title + '"', article.guid); } }); return deduped_articles; diff --git a/lib/example_plugins/content_encoded.js b/lib/example_plugins/content_encoded.js deleted file mode 100644 index d9b477f..0000000 --- a/lib/example_plugins/content_encoded.js +++ /dev/null @@ -1,20 +0,0 @@ -// Put the description into content:encoded block -// Ex: -// -// Stewart let the news slip during a taping of his show today.]]> -// -module.exports = function (item, itemOptions, source) { - if (!item || !itemOptions) { - return; - } - if (item["content:encoded"] && item["content:encoded"]["#"]){ - var content_encoded = item["content:encoded"]["#"]; - itemOptions.custom_elements.push( - { "content:encoded": - { - _cdata: content_encoded - } - } - ); - } -}; \ No newline at end of file diff --git a/package.json b/package.json index 9f50bfa..3fdd888 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "rss-braider", - "version": "0.1.5", + "version": "1.0.0", "description": "Braid/aggregate/combine RSS feeds into a single RSS (or JSON) document. Optionally process through specified plugins.", "main": "index.js", "repository": "https://github.com/KQED/rss-braider", @@ -41,7 +41,7 @@ "rss": "git://github.com/rv-kip/node-rss.git#8d1420" }, "devDependencies": { - "tape": "^4.0.0", - "mockdate": "^1.0.3" + "mockdate": "^1.0.3", + "tape": "^4.0.0" } } diff --git a/test/expected_output/emptyFeed.xml b/test/expected_output/emptyFeed.xml new file mode 100644 index 0000000..169bd60 --- /dev/null +++ b/test/expected_output/emptyFeed.xml @@ -0,0 +1,10 @@ + + + + <![CDATA[A Feed with no elements]]> + + http://github.com/dylang/node-rss + rss-braider + Wed, 31 Dec 2014 00:00:01 GMT + + \ No newline at end of file diff --git a/test/expected_output/fileFeedBadPlugin.xml b/test/expected_output/fileFeedBadPlugin.xml new file mode 100644 index 0000000..ad5ffb9 --- /dev/null +++ b/test/expected_output/fileFeedBadPlugin.xml @@ -0,0 +1,35 @@ + + + + <![CDATA[A Feed with no elements]]> + + http://github.com/dylang/node-rss + rss-braider + Wed, 31 Dec 2014 00:00:01 GMT + + <![CDATA[Rent Hike For Dance Mission Theater Has Artists Worried About Uncertain Future]]> + Stepping out of BART at 24th and Mission at most hours of the day, one is likely to hear the pulse of African drums, hip-hop or salsa emanating from the second-floor studios of Dance Brigade's Dance Mission Theater. But that music may not continue forever.

+

The performance space and dance school ...read more]]> + http://ww2.kqed.org/news/2014/12/20/dance-mission-theater-rent-increase-worries-artists/ + http://ww2.kqed.org/arts/2014/12/20/rent-hike-for-dance-mission-theater-has-artists-worried-about-uncertain-future/ + + Sat, 20 Dec 2014 09:00:22 GMT + + + <![CDATA[Bob Miller: Teasing Science Lessons from Everyday Phenomena]]> + + http://ww2.kqed.org/arts/2014/12/20/bob-miller-teasing-science-lessons-from-everyday-phenomena/ + http://ww2.kqed.org/arts/?p=10220517 + + Sat, 20 Dec 2014 14:00:47 GMT + + + <![CDATA[Light Art Brings Holiday Glow to Darkest Nights]]> + + http://ww2.kqed.org/arts/2014/12/21/on-darkest-nights-illuminated-art-brings-holiday-glow/ + http://ww2.kqed.org/arts/?p=10231062 + + Sun, 21 Dec 2014 14:00:08 GMT + + + \ No newline at end of file diff --git a/test/feeds/date_sort.js b/test/feeds/date_sort.js index ad64076..1763bbc 100644 --- a/test/feeds/date_sort.js +++ b/test/feeds/date_sort.js @@ -2,7 +2,7 @@ var feed = { "feed_name" : "test file feed", "default_count" : 1, "no_cdata_fields" : ['description'], - "plugins" : ['kqed', 'content_encoded', 'wfw_slash_comments', 'add_media_thumbnail'], + "plugins" : ['kqed', 'add_content_encoded_block', 'wfw_slash_comments', 'add_media_thumbnail'], "meta" : { "title": "Test File Feed", "description": "This feed comes from a file", diff --git a/test/feeds/no_elements.js b/test/feeds/no_elements.js new file mode 100644 index 0000000..e78a598 --- /dev/null +++ b/test/feeds/no_elements.js @@ -0,0 +1,18 @@ +var feed = { + "feed_name" : "no_elements", + "plugins" : ['filter_out_all_articles'], // No articles make it through + "meta" : { + "title" : "A Feed with no elements", + "description" : "This feed will have no elements as a result of the filter_all_articles plugin. Used for unit tests.", + "url" : "http://rss.nytimes.com/services/xml/rss/nyt/Technology.xml", + }, + "sources" : [ + { + "name" : "nyt_tech", + "feed_url" : "http://rss.nytimes.com/services/xml/rss/nyt/Technology.xml", + "count" : 5, + "fullname" : "NYT Technology" + } + ] +}; +exports.feed = feed; \ No newline at end of file diff --git a/test/feeds/sample_feed.js b/test/feeds/sample_feed.js index e9d5e08..8d9d495 100644 --- a/test/feeds/sample_feed.js +++ b/test/feeds/sample_feed.js @@ -16,11 +16,10 @@ var feed = { }, "sources" : [ { - "name" : "sample_feed", + "name" : "sample_source", "count" : 1, "file_path" : __dirname + "/../input_files/sample_feed.xml", - }, - + } ] }; exports.feed = feed; \ No newline at end of file diff --git a/test/feeds/sample_feed_bad_plugin.js b/test/feeds/sample_feed_bad_plugin.js new file mode 100644 index 0000000..07f3088 --- /dev/null +++ b/test/feeds/sample_feed_bad_plugin.js @@ -0,0 +1,17 @@ +var feed = { + "feed_name" : "no_elements", + // "plugins" : ['bad_plugin'], // Intentionally bad plugin for testing + "meta" : { + "title" : "A Feed with no elements", + "description" : "This feed will have no elements as a result of the filter_all_articles plugin. Used for unit tests.", + "url" : "http://rss.nytimes.com/services/xml/rss/nyt/Technology.xml", + }, + "sources" : [ + { + "name" : "sample_source", + "count" : 3, + "file_path" : __dirname + "/../input_files/sample_feed.xml", + } + ] +}; +exports.feed = feed; \ No newline at end of file diff --git a/test/feeds/sample_feed_duplicates.js b/test/feeds/sample_feed_duplicates.js index 79124ed..e7f737a 100644 --- a/test/feeds/sample_feed_duplicates.js +++ b/test/feeds/sample_feed_duplicates.js @@ -2,7 +2,7 @@ var feed = { "feed_name" : "test file feed", "default_count" : 1, "no_cdata_fields" : ['description'], - "plugins" : ['kqed', 'content_encoded', 'wfw_slash_comments', 'add_media_thumbnail'], + "plugins" : ['kqed', 'add_content_encoded_block', 'wfw_slash_comments', 'add_media_thumbnail'], "meta" : { "title": "Test File Feed", "description": "This feed comes from a file", diff --git a/test/feeds/sample_feed_plugins.js b/test/feeds/sample_feed_plugins.js index 25f8a7f..3487917 100644 --- a/test/feeds/sample_feed_plugins.js +++ b/test/feeds/sample_feed_plugins.js @@ -2,7 +2,7 @@ var feed = { "feed_name" : "test file feed", "default_count" : 1, "no_cdata_fields" : ['description'], - "plugins" : ['kqed', 'content_encoded', 'wfw_slash_comments', 'add_media_thumbnail'], + "plugins" : ['kqed', 'add_content_encoded_block', 'wfw_slash_comments', 'add_media_thumbnail'], "meta" : { "title": "Test File Feed", "description": "This feed comes from a file", diff --git a/test/index.js b/test/index.js index 8094985..21c7ab5 100644 --- a/test/index.js +++ b/test/index.js @@ -6,7 +6,7 @@ var test = require('tape'), // lastBuildDate will always be this value var mockdate = require('mockdate').set('Wed, 31 Dec 2014 00:00:01 GMT'); -test('braid feed from file without plugins', function(t) { +test('generate feed. No plugins', function(t) { t.plan(1); var feeds = {}; feeds.sample_feed = require("./feeds/sample_feed").feed; @@ -27,7 +27,7 @@ test('braid feed from file without plugins', function(t) { }); }); -test('braid feed from file with plugins', function(t) { +test('generate feed and process through plugins', function(t) { t.plan(1); var feeds = {}; feeds.sample_feed = require("./feeds/sample_feed_plugins").feed; @@ -35,7 +35,7 @@ test('braid feed from file with plugins', function(t) { feeds : feeds, indent : " ", date_sort_order : "desc", - plugins_directories : [__dirname + '/../lib/example_plugins/'] + plugins_directories : [__dirname + '/../examples/plugins/'] }; var rss_braider = RssBraider.createClient(braider_options); @@ -48,7 +48,7 @@ test('braid feed from file with plugins', function(t) { }); }); -test('deduplicate feed from file', function(t) { +test('de-duplicate feed', function(t) { t.plan(1); var feeds = {}; feeds.sample_feed = require("./feeds/sample_feed_duplicates").feed; @@ -56,9 +56,10 @@ test('deduplicate feed from file', function(t) { feeds : feeds, indent : " ", dedupe_fields : ["title", "guid"], - plugins_directories : [__dirname + '/../lib/example_plugins/'] + plugins_directories : [__dirname + '/../examples/plugins/'] }; var rss_braider = RssBraider.createClient(braider_options); + rss_braider.logger.level('info'); rss_braider.processFeed('sample_feed', 'rss', function(err, data){ if (err) { @@ -69,7 +70,7 @@ test('deduplicate feed from file', function(t) { }); }); -test('sort by date desc', function(t) { +test('sort feed articles by date descending', function(t) { t.plan(1); var feeds = {}; feeds.sample_feed = require("./feeds/date_sort").feed; @@ -77,7 +78,7 @@ test('sort by date desc', function(t) { feeds : feeds, indent : " ", date_sort_order : "desc", - plugins_directories : [__dirname + '/../lib/example_plugins/'] + plugins_directories : [__dirname + '/../examples/plugins/'] }; var rss_braider = RssBraider.createClient(braider_options); @@ -90,7 +91,7 @@ test('sort by date desc', function(t) { }); }); -test('sort by date asc', function(t) { +test('sort feed articles by date ascending', function(t) { t.plan(1); var feeds = {}; feeds.sample_feed = require("./feeds/date_sort").feed; @@ -98,7 +99,7 @@ test('sort by date asc', function(t) { feeds : feeds, indent : " ", date_sort_order : "asc", - plugins_directories : [__dirname + '/../lib/example_plugins/'] + plugins_directories : [__dirname + '/../examples/plugins/'] }; var rss_braider = RssBraider.createClient(braider_options); @@ -111,4 +112,46 @@ test('sort by date asc', function(t) { }); }); +test('filter all articles out using plugin', function(t) { + t.plan(1); + var feeds = {}; + feeds.sample_feed = require("./feeds/no_elements").feed; + var braider_options = { + feeds : feeds, + indent : " ", + date_sort_order : "asc", + plugins_directories : [__dirname + '/../examples/plugins/'] + }; + var rss_braider = RssBraider.createClient(braider_options); + rss_braider.logger.level('info'); + rss_braider.processFeed('sample_feed', 'rss', function(err, data){ + if (err) { + return t.fail(err); + } + // console.log(data); + t.equal(data, expectedOutput.emptyFeed); + }); +}); + +test("Don't break when a filter fails and returns null", function(t) { + t.plan(1); + var feeds = {}; + feeds.sample_feed = require("./feeds/sample_feed_bad_plugin").feed; + var braider_options = { + feeds : feeds, + indent : " ", + date_sort_order : "asc", + plugins_directories : [__dirname + '/../examples/plugins/'] + }; + var rss_braider = RssBraider.createClient(braider_options); + rss_braider.logger.level('info'); + + rss_braider.processFeed('sample_feed', 'rss', function(err, data){ + if (err) { + return t.fail(err); + } + // console.log(data); + t.equal(data, expectedOutput.fileFeedBadPlugin); + }); +});