martin/rss-braider
1
0
Fork 0
mirror of https://gitlab.silvrtree.co.uk/martind2000/rss-braider.git synced 2025-02-11 06:09:19 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #4 from KQED/RSS-26-deleting-items

Browse Source 
RSS26 - Plugin refactor to support filtering out of items
This commit is contained in:
Kip Gebhardt 2015-06-12 15:00:48 -07:00
commit b6abb56f0b
25 changed files with 302 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
README.md
View File

@ -16,7 +16,7 @@ $ cd examples
$ node simple.js (combines 3 sources)
$ node plugins.js (combines 3 sources and runs a transformation plugin)
```
## Code Example
### Code Example
```js
var RssBraider = require('rss-braider'),
feeds = {};
@ -48,10 +48,14 @@ feeds.simple_test_feed = {
var braider_options = {
feeds : feeds,
indent : " ",
date_sort_order : "desc" // Newest first
date_sort_order : "desc", // Newest first
log_level : "debug"
};
var rss_braider = RssBraider.createClient(braider_options);
// Override logging level (debug, info, warn, err, off)
rss_braider.logger.level('off');
// Output braided feed as rss. use 'json' for JSON output.
rss_braider.processFeed('simple_test_feed', 'rss', function(err, data){
if (err) {
@ -59,4 +63,40 @@ rss_braider.processFeed('simple_test_feed', 'rss', function(err, data){
}
console.log(data);
});
```
```
## Plugins
Plugins provide custom manipulation and filtering of RSS items/articles. See `examples/plugins` for examples.
A plugin operates by modifying the `itemOptions` object or by returning `null` which will exclude the `item` (article) from the resulting feed (See `examples/plugins/filter_out_all_articles.js`).
The `itemsOptions` object gets passed to `node-rss` to generate the RSS feeds, so read the documentation on that module and its use of custom namespaces. (https://github.com/dylang/node-rss)
### Plugin Example
This plugin will capitalize the article title for all articles
```js
module.exports = function (item, itemOptions, source) {
if (!item || !itemOptions) {
return;
}
if (itemOptions.title) {
itemOptions.title = itemOptions.title.toUpperCase();
}
return itemOptions;
};
```
The plugin is registered with the feed in the feed config .js file and are run in order.
```js
var feed = {
"feed_name" : "feed with plugins",
"default_count" : 1,
"plugins" : ['capitalize_title', 'plugin_template'],
...
```
## Release Notes
### 1.0.0
Changed plugin architecture to allow filtering out of article/items by returning `-1` instead of a modified `itemsOptions` object. This is a breaking change as it will require existing plugins to return `itemsOptions` instead of modifying the reference. See `examples/plugins`.

2
examples/config/feed_with_plugins.js
View File

@ -2,7 +2,7 @@ var feed = {
"feed_name" : "feed with plugins",
"default_count" : 1,
"no_cdata_fields" : [],
"plugins" : ['content_encoded'],
"plugins" : ['capitalize_title', 'plugin_template'],
"meta" : {
"title": "NPR Braided Feed",
"description": "This is a test of two NPR sources from file. Plugins are applied."

4
examples/plugins/content_encoded.js → examples/plugins/add_content_encoded_block.js
View File

@ -4,9 +4,6 @@
// <![CDATA[<p>Stewart let the news slip during a taping of his show today.]]>
// </content:encoded>
module.exports = function (item, itemOptions, source) {
if (!item || !itemOptions) {
return;
}
if (item["content:encoded"] && item["content:encoded"]["#"]){
var content_encoded = item["content:encoded"]["#"];
itemOptions.custom_elements.push(
@ -17,4 +14,5 @@ module.exports = function (item, itemOptions, source) {
}
);
}
return itemOptions;
};

48
lib/example_plugins/strip_elements_from_powerpress.js → examples/plugins/add_itunes_elements.js
View File

@ -1,27 +1,5 @@
module.exports = function (item, itemOptions, source) {
if (!item || !itemOptions) {
return;
}
// 'itunes:summary':
// { '@': {},
// '#': 'Let KQED Arts help you find the best things to see, do, and explore in San Francisco, Oakland, San Jose, and the surrounding areas.' },
// 'itunes:author': { '@': {}, '#': 'KQED Arts' },
// 'itunes:explicit': { '@': {}, '#': 'no' },
// 'itunes:image': { '@': [Object] },
// 'itunes:owner': { '@': {}, 'itunes:name': [Object], 'itunes:email': [Object] },
// 'rss:managingeditor':
// { '@': {},
// '#': 'ondemand@kqed.org (KQED Arts)',
// name: 'KQED Arts',
// email: 'ondemand@kqed.org' },
// 'rss:copyright':
// { '@': {},
// '#': 'Copyright © 2015 KQED Inc. All Rights Reserved.' },
// 'itunes:subtitle': { '@': {}, '#': 'KQED Public Media for Northern CA' },
// 'rss:image': { '@': {}, title: [Object], url: [Object], link: [Object] },
// 'itunes:category': [ [Object], [Object], [Object] ],
// Pass through itunes content...
//
// <itunes:summary>
// Let KQED Arts help you find the best things to see, do, and explore in San Francisco, Oakland, San Jose, and the surrounding areas.
// </itunes:summary>
@ -33,28 +11,9 @@ module.exports = function (item, itemOptions, source) {
// <itunes:email>ondemand@kqed.org</itunes:email>
// </itunes:owner>
// <managingEditor>ondemand@kqed.org (KQED Arts)</managingEditor>
// <copyright>Copyright © 2015 KQED Inc. All Rights Reserved.</copyright>
// <itunes:subtitle>KQED Public Media for Northern CA</itunes:subtitle>
// *link
// *comments
// pubDate
// *dc:creator
// *category
// guid
// description
// *content:encoded
// *wfw:commentRss
// *slash:comments
// enclosure
// itunes:subtitle
// itunes:summary
// itunes:author
// itunes:explicit
// *media:content (multiple)
// *media:thumbnail
// Pass through itunes content
module.exports = function (item, itemOptions, source) {
var pass_through_arr = ['itunes:summary', 'itunes:author', 'itunes:explicit', ];
pass_through_arr.forEach(function(element){
if (item[element] && item[element]['#']) {
@ -82,4 +41,5 @@ module.exports = function (item, itemOptions, source) {
}
});
}
return itemOptions;
};

5
lib/example_plugins/add_media_thumbnail.js → examples/plugins/add_media_thumbnail.js
View File

@ -6,10 +6,6 @@
// 'media:thumbnail'
var _ = require('lodash');
module.exports = function (item, itemOptions, source) {
if (!item || !itemOptions) {
return;
}
var thumbnail;
if (item['media:thumbnail'] && item['media:thumbnail']['#']) {
thumbnail = {
@ -51,4 +47,5 @@ module.exports = function (item, itemOptions, source) {
}
}
}
return itemOptions;
};

8
examples/plugins/bad_plugin.js Normal file
View File

@ -0,0 +1,8 @@
module.exports = function (item, itemOptions, source) {
// This is an intentionally broken plugin for testing.
// This doesn't do anything and shouldn't be a template
// for other plugins
return;
};

7
examples/plugins/capitalize_title.js Normal file
View File

@ -0,0 +1,7 @@
module.exports = function (item, itemOptions, source) {
if (itemOptions.title) {
itemOptions.title = itemOptions.title.toUpperCase();
}
return itemOptions;
};

8
examples/plugins/filter_out_all_articles.js Normal file
View File

@ -0,0 +1,8 @@
module.exports = function (item, itemOptions, source) {
if (!item || !itemOptions) {
return;
}
// This plugin removes all items by returning -1 instead of the processed itemOptions
return -1;
};

4
lib/example_plugins/kqed.js → examples/plugins/kqed.js
View File

@ -1,8 +1,5 @@
// define kqed source
module.exports = function (item, itemOptions, source) {
if (!item || !itemOptions || !source) {
return;
}
// Look for kqed namespace elements in source and add as custom elements for item
// Ex:
// <kqed:fullname>The California Report</kqed:fullname>
@ -36,4 +33,5 @@ module.exports = function (item, itemOptions, source) {
{ 'kqed:feed_url': item.feed_url }
);
}
return itemOptions;
};

6
examples/plugins/plugin_template.js Normal file
View File

@ -0,0 +1,6 @@
module.exports = function (item, itemOptions, source) {
// This plugin does no processing
// It's just a template
return itemOptions;
};

4
lib/example_plugins/wfw_slash_comments.js → examples/plugins/wfw_slash_comments.js
View File

@ -1,7 +1,4 @@
module.exports = function (item, itemOptions, source) {
if (!item || !itemOptions) {
return;
}
// wfw
if (item["wfw:commentrss"] && item["wfw:commentrss"]["#"]){
itemOptions.custom_elements.push({ "wfw:commentRss": item["wfw:commentrss"]["#"]});
@ -11,4 +8,5 @@ module.exports = function (item, itemOptions, source) {
if (item["slash:comments"] && item["slash:comments"]["#"]){
itemOptions.custom_elements.push({ "slash:comments": item["slash:comments"]["#"]});
}
return itemOptions;
};

6
examples/simple.js
View File

@ -28,10 +28,14 @@ feeds.simple_test_feed = {
var braider_options = {
feeds : feeds,
indent : " ",
date_sort_order : "desc" // Newest first
date_sort_order : "desc", // Newest first
log_level : 'debug'
};
var rss_braider = RssBraider.createClient(braider_options);
// Set logging level (debug, info, warn, err, off)
rss_braider.logger.level('off');
rss_braider.processFeed('simple_test_feed', 'rss', function(err, data){
if (err) {
return console.log(err);

3
examples/plugins.js → examples/use_plugins.js
View File

@ -6,7 +6,8 @@ feed_obj.filefeed = require("./config/feed_with_plugins").feed;
var braider_options = {
feeds : feed_obj,
indent : " ",
plugins_directories : [__dirname + "/plugins/"]
plugins_directories : [__dirname + "/plugins/"],
log_level : 'debug'
};
var rss_braider = RssBraider.createClient(braider_options);

123
lib/RssBraider.js
View File

@ -1,26 +1,32 @@
// process feed-reader item into node-rss item
var FeedParser = require('feedparser'),
bunyan = require('bunyan'),
_ = require('lodash'),
async = require('async'),
request = require('request'),
RSS = require('rss'),
fs = require('fs'),
package_json = require('../package.json'),
logger;
var FeedParser = require('feedparser'),
bunyan = require('bunyan'),
_ = require('lodash'),
async = require('async'),
request = require('request'),
RSS = require('rss'),
fs = require('fs');
var logger;
var RssBraider = function (options) {
if (!options) {
options = {};
}
this.feeds = options.feeds || null;
this.logger = logger = options.logger || bunyan.createLogger({name: 'rss-braider'});
this.logger = options.logger || bunyan.createLogger({name: package_json.name});
if (options.log_level) {
this.logger.level(options.log_level);
}
this.indent = options.indent || " ";
this.dedupe_fields = options.dedupe_fields || []; // The fields to use to identify duplicate articles
this.date_sort_order = options.date_sort_order || "desc";
this.plugins_directories = options.plugins_directories || [];
// load plugins from plugins folder
// TODO, specify plugins location
this.plugins = {};
this.loadPlugins();
};
// loadup self.plugins with the plugin functions
@ -28,7 +34,7 @@ RssBraider.prototype.loadPlugins = function () {
var self = this;
if (self.plugins_directories.length < 1) {
// logger.info("No plugins_directories specified. No plugins loaded.");
self.logger.debug("No plugins_directories specified. No plugins loaded.");
}
self.plugins_directories.forEach(function(path){
// load up each file and assign it to the plugins
@ -36,10 +42,10 @@ RssBraider.prototype.loadPlugins = function () {
filenames.forEach(function(filename){
var plugin_name = filename.replace(/.js$/, '');
if (self.plugins[plugin_name]) {
logger.warn("Duplicate plugin name: ", plugin_name, "Overwriting with newer plugin");
self.logger.warn("Duplicate plugin name: ", plugin_name, "Overwriting with newer plugin");
}
self.plugins[plugin_name] = require(path + '/' + plugin_name);
// logger.info("plugin registered:", plugin_name);
self.logger.debug("plugin registered:", plugin_name);
});
});
};
@ -56,35 +62,32 @@ RssBraider.prototype.feedExists = function (feed_name) {
// trim down to desired count, dedupe and sort
RssBraider.prototype.processFeed = function(feed_name, format, callback)
{
if (!format) {
format = 'rss';
}
var self = this,
feed = self.feeds[feed_name],
feed_articles = [];
// logger.info("DEBUG processFeed: feed is set to " + feed_name);
if (!format) {
format = 'rss';
}
if (!feed || !feed.sources || feed.sources.length < 1) {
return callback("No definition for feed name: " + feed_name);
}
// Process each feed source through Feedparser to get articles.
// Then process each item/article through rss-braider and any plugins
async.each(feed.sources, function(source, callback) {
var count = source.count || feed.default_count || 10, // Number of articles
var count = source.count || feed.default_count || 10, // Number of articles per source
url = source.feed_url || null,
file_path = source.file_path || null,
source_articles = [];
logger.debug("Requesting source:" + source.name + " at " + url + " for feed:" + feed_name);
// todo: Check if source.file is set and set up a fs stream read
var feedparser = new FeedParser();
if (url) {
var req = request(url);
// logger.info("request to", url);
req.on('error', function (error) {
logger.error(error);
self.logger.error(error);
});
req.on('response', function (res) {
@ -99,26 +102,27 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
var filestream = fs.createReadStream(file_path);
filestream.pipe(feedparser);
} else {
logger.error("url or file_path not defined for feed: " + source.name);
self.logger.error("url or file_path not defined for feed: " + source.name);
return callback();
}
feedparser.on('error', function(error) {
logger.error("feedparser error:", error, "name:", source.name, "source:", source.feed_url);
self.logger.error("feedparser",", source.name:", source.name, ", url:", source.feed_url, error.stack);
});
// Collect the articles from this source
feedparser.on('readable', function() {
// This is where the action is!
var stream = this,
item;
while ( item = stream.read() ) {
while ( !!(item = stream.read()) ) {
if (source.feed_url) {
item.source_url = source.feed_url;
}
// Process Item/Article
var article = self.processItem(item, source, feed_name);
// plugins may filter items and return null
if (article) {
source_articles.push(article);
}
@ -126,7 +130,7 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
});
feedparser.on("end", function(){
// sort and de-dupe this feed's articles and push them into array
// de-dupe , date sort, and trim this feed's articles and push them into array
source_articles = self.dedupe(source_articles, self.dedupe_fields);
source_articles = self.date_sort(source_articles);
source_articles = source_articles.slice(0, count);
@ -136,14 +140,14 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
},
function(err){
if (err) {
logger.error(err);
self.logger.error(err);
return callback(err);
} else {
// Final Dedupe step and resort
feed_articles = self.dedupe(feed_articles, self.dedupe_fields);
feed_articles = self.date_sort(feed_articles);
// Create new feed with these articles
// Create new feed with these articles. Follows node-rss spec
var options = {
title : feed.meta.title,
description : feed.meta.description,
@ -166,11 +170,10 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
ret_string = JSON.stringify(newfeed);
break;
case 'rss':
case 'xml':
ret_string = newfeed.xml(self.indent);
break;
default:
logger.error("Unknown format:", format);
self.logger.error("Unknown format:", format);
ret_string = "{}";
}
@ -183,8 +186,8 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
RssBraider.prototype.processItem = function (item, source, feed_name) {
var self = this;
if (!item) {
logger.error("processItem: no item passed in");
if (!item || !source || !feed_name) {
self.logger.error("processItem: missing item, source, and/or feed_name");
return null;
}
// Basics
@ -201,26 +204,46 @@ RssBraider.prototype.processItem = function (item, source, feed_name) {
};
// Run the plugins specified by the "plugins" section of the
// feed config file to build out any custom elements or
// do transforms
self.runPlugins(item, itemOptions, source, feed_name);
// feed .js file to build out any custom elements or
// do transforms/filters
var filteredItemOptions = self.runPlugins(item, itemOptions, source, feed_name);
return itemOptions;
return filteredItemOptions;
};
RssBraider.prototype.runPlugins = function (item, itemOptions, source, feed_name) {
var self = this,
feed = self.feeds[feed_name] || {},
plugins_list = feed.plugins || [];
plugins_list = feed.plugins || [],
ret_val,
filteredItemOptions;
// Process the item through the desired feed plugins
plugins_list.forEach(function(plugin_name){
// plugins_list.forEach(function(plugin_name){
for (var i = 0; i < plugins_list.length; i++) {
var plugin_name = plugins_list[i];
if (self.plugins[plugin_name]) {
// logger.info("DEBUG runPlugins running " + plugin_name + " for item " + item.guid + " in feed: " + feed.meta.title);
self.plugins[plugin_name](item, itemOptions, source);
filteredItemOptions = self.plugins[plugin_name](item, itemOptions, source);
} else {
logger.error("A plugin named '" + plugin_name + "' hasn't been registered");
self.logger.error("A plugin named '" + plugin_name + "' hasn't been registered");
}
});
// A plugin returning -1 means skip this item
if (filteredItemOptions === -1) {
self.logger.debug("Plugin '" + plugin_name + "' filtered item from feed '" + feed.meta.title + "'", item.guid);
itemOptions = null;
break;
}
// Check that the plugin didn't just return null or undef, which would be bad.
if (!filteredItemOptions) {
self.logger.debug("Plugin '" + plugin_name + "' failed to return itemOptions for feed:'" + feed.meta.title + "'", item.guid);
filteredItemOptions = itemOptions; // Reset
}
// Prepare for next plugin.
itemOptions = filteredItemOptions;
}
return itemOptions;
};
// Dedupe articles in node-rss itemOptions format
@ -228,6 +251,7 @@ RssBraider.prototype.runPlugins = function (item, itemOptions, source, feed_name
// operation on the articles array
// TODO, make this a plugin?
RssBraider.prototype.dedupe = function(articles_arr, fields){
var self = this;
if ( !fields || fields.length < 1 ) {
return _.uniq(articles_arr);
} else {
@ -249,8 +273,9 @@ RssBraider.prototype.dedupe = function(articles_arr, fields){
// it's unique
deduped_articles.push(article);
} else {
// The article matched all of another article's fields
// Do nothing
// The article matched all of another article's "dedupe" fields
// so filter it out (i.e. do nothing)
self.logger.debug("skipping duplicate", '"' + article.title + '"', article.guid);
}
});
return deduped_articles;

20
lib/example_plugins/content_encoded.js
View File

@ -1,20 +0,0 @@
// Put the description into content:encoded block
// Ex:
// <content:encoded>
// <![CDATA[<p>Stewart let the news slip during a taping of his show today.]]>
// </content:encoded>
module.exports = function (item, itemOptions, source) {
if (!item || !itemOptions) {
return;
}
if (item["content:encoded"] && item["content:encoded"]["#"]){
var content_encoded = item["content:encoded"]["#"];
itemOptions.custom_elements.push(
{ "content:encoded":
{
_cdata: content_encoded
}
}
);
}
};

6
package.json
View File

@ -1,6 +1,6 @@
{
"name": "rss-braider",
"version": "0.1.5",
"version": "1.0.0",
"description": "Braid/aggregate/combine RSS feeds into a single RSS (or JSON) document. Optionally process through specified plugins.",
"main": "index.js",
"repository": "https://github.com/KQED/rss-braider",
@ -41,7 +41,7 @@
"rss": "git://github.com/rv-kip/node-rss.git#8d1420"
},
"devDependencies": {
"tape": "^4.0.0",
"mockdate": "^1.0.3"
"mockdate": "^1.0.3",
"tape": "^4.0.0"
}
}

10
test/expected_output/emptyFeed.xml Normal file
View File

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:atom="http://www.w3.org/2005/Atom" version="2.0">
<channel>
<title><![CDATA[A Feed with no elements]]></title>
<description><![CDATA[This feed will have no elements as a result of the filter_all_articles plugin. Used for unit tests.]]></description>
<link>http://github.com/dylang/node-rss</link>
<generator>rss-braider</generator>
<lastBuildDate>Wed, 31 Dec 2014 00:00:01 GMT</lastBuildDate>
</channel>
</rss>

35
test/expected_output/fileFeedBadPlugin.xml Normal file
View File

@ -0,0 +1,35 @@
<?xml version="1.0" encoding="UTF-8"?>
<rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:atom="http://www.w3.org/2005/Atom" version="2.0">
<channel>
<title><![CDATA[A Feed with no elements]]></title>
<description><![CDATA[This feed will have no elements as a result of the filter_all_articles plugin. Used for unit tests.]]></description>
<link>http://github.com/dylang/node-rss</link>
<generator>rss-braider</generator>
<lastBuildDate>Wed, 31 Dec 2014 00:00:01 GMT</lastBuildDate>
<item>
<title><![CDATA[Rent Hike For Dance Mission Theater Has Artists Worried About Uncertain Future]]></title>
<description><![CDATA[<p>Stepping out of BART at 24th and Mission at most hours of the day, one is likely to hear the pulse of African drums, hip-hop or salsa emanating from the second-floor studios of Dance Brigade's Dance Mission Theater. But that music may not continue forever.</p>
<p>The performance space and dance school <a href="http://ww2.kqed.org/news/2014/12/20/dance-mission-theater-rent-increase-worries-artists/" target="_self" id="rssmi_more"> ...read more</a>]]></description>
<link>http://ww2.kqed.org/news/2014/12/20/dance-mission-theater-rent-increase-worries-artists/</link>
<guid isPermaLink="false">http://ww2.kqed.org/arts/2014/12/20/rent-hike-for-dance-mission-theater-has-artists-worried-about-uncertain-future/</guid>
<dc:creator><![CDATA[KQED Arts]]></dc:creator>
<pubDate>Sat, 20 Dec 2014 09:00:22 GMT</pubDate>
</item>
<item>
<title><![CDATA[Bob Miller: Teasing Science Lessons from Everyday Phenomena]]></title>
<description><![CDATA[Until February 5, 2015, you can visit the main branch of the San Francisco Public Library and be momentarily transported through space and time to the early days of the Exploratorium via the life and work of Bob Miller, who died in 2007.]]></description>
<link>http://ww2.kqed.org/arts/2014/12/20/bob-miller-teasing-science-lessons-from-everyday-phenomena/</link>
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10220517</guid>
<dc:creator><![CDATA[Sarah Hotchkiss]]></dc:creator>
<pubDate>Sat, 20 Dec 2014 14:00:47 GMT</pubDate>
</item>
<item>
<title><![CDATA[Light Art Brings Holiday Glow to Darkest Nights]]></title>
<description><![CDATA[In the dark of winter, San Franciscans with an urge to celebrate the light can visit a new wealth of illuminated art installations. This video tour offers a preview of some of the more dazzling works.]]></description>
<link>http://ww2.kqed.org/arts/2014/12/21/on-darkest-nights-illuminated-art-brings-holiday-glow/</link>
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10231062</guid>
<dc:creator><![CDATA[KQED Arts]]></dc:creator>
<pubDate>Sun, 21 Dec 2014 14:00:08 GMT</pubDate>
</item>
</channel>
</rss>

2
test/feeds/date_sort.js
View File

@ -2,7 +2,7 @@ var feed = {
"feed_name" : "test file feed",
"default_count" : 1,
"no_cdata_fields" : ['description'],
"plugins" : ['kqed', 'content_encoded', 'wfw_slash_comments', 'add_media_thumbnail'],
"plugins" : ['kqed', 'add_content_encoded_block', 'wfw_slash_comments', 'add_media_thumbnail'],
"meta" : {
"title": "Test File Feed",
"description": "This feed comes from a file",

18
test/feeds/no_elements.js Normal file
View File

@ -0,0 +1,18 @@
var feed = {
"feed_name" : "no_elements",
"plugins" : ['filter_out_all_articles'], // No articles make it through
"meta" : {
"title" : "A Feed with no elements",
"description" : "This feed will have no elements as a result of the filter_all_articles plugin. Used for unit tests.",
"url" : "http://rss.nytimes.com/services/xml/rss/nyt/Technology.xml",
},
"sources" : [
{
"name" : "nyt_tech",
"feed_url" : "http://rss.nytimes.com/services/xml/rss/nyt/Technology.xml",
"count" : 5,
"fullname" : "NYT Technology"
}
]
};
exports.feed = feed;

5
test/feeds/sample_feed.js
View File

@ -16,11 +16,10 @@ var feed = {
},
"sources" : [
{
"name" : "sample_feed",
"name" : "sample_source",
"count" : 1,
"file_path" : __dirname + "/../input_files/sample_feed.xml",
},
}
]
};
exports.feed = feed;

17
test/feeds/sample_feed_bad_plugin.js Normal file
View File

@ -0,0 +1,17 @@
var feed = {
"feed_name" : "no_elements",
// "plugins" : ['bad_plugin'], // Intentionally bad plugin for testing
"meta" : {
"title" : "A Feed with no elements",
"description" : "This feed will have no elements as a result of the filter_all_articles plugin. Used for unit tests.",
"url" : "http://rss.nytimes.com/services/xml/rss/nyt/Technology.xml",
},
"sources" : [
{
"name" : "sample_source",
"count" : 3,
"file_path" : __dirname + "/../input_files/sample_feed.xml",
}
]
};
exports.feed = feed;

2
test/feeds/sample_feed_duplicates.js
View File

@ -2,7 +2,7 @@ var feed = {
"feed_name" : "test file feed",
"default_count" : 1,
"no_cdata_fields" : ['description'],
"plugins" : ['kqed', 'content_encoded', 'wfw_slash_comments', 'add_media_thumbnail'],
"plugins" : ['kqed', 'add_content_encoded_block', 'wfw_slash_comments', 'add_media_thumbnail'],
"meta" : {
"title": "Test File Feed",
"description": "This feed comes from a file",

2
test/feeds/sample_feed_plugins.js
View File

@ -2,7 +2,7 @@ var feed = {
"feed_name" : "test file feed",
"default_count" : 1,
"no_cdata_fields" : ['description'],
"plugins" : ['kqed', 'content_encoded', 'wfw_slash_comments', 'add_media_thumbnail'],
"plugins" : ['kqed', 'add_content_encoded_block', 'wfw_slash_comments', 'add_media_thumbnail'],
"meta" : {
"title": "Test File Feed",
"description": "This feed comes from a file",

61
test/index.js
View File

@ -6,7 +6,7 @@ var test = require('tape'),
// lastBuildDate will always be this value
var mockdate = require('mockdate').set('Wed, 31 Dec 2014 00:00:01 GMT');
test('braid feed from file without plugins', function(t) {
test('generate feed. No plugins', function(t) {
t.plan(1);
var feeds = {};
feeds.sample_feed = require("./feeds/sample_feed").feed;
@ -27,7 +27,7 @@ test('braid feed from file without plugins', function(t) {
});
});
test('braid feed from file with plugins', function(t) {
test('generate feed and process through plugins', function(t) {
t.plan(1);
var feeds = {};
feeds.sample_feed = require("./feeds/sample_feed_plugins").feed;
@ -35,7 +35,7 @@ test('braid feed from file with plugins', function(t) {
feeds : feeds,
indent : " ",
date_sort_order : "desc",
plugins_directories : [__dirname + '/../lib/example_plugins/']
plugins_directories : [__dirname + '/../examples/plugins/']
};
var rss_braider = RssBraider.createClient(braider_options);
@ -48,7 +48,7 @@ test('braid feed from file with plugins', function(t) {
});
});
test('deduplicate feed from file', function(t) {
test('de-duplicate feed', function(t) {
t.plan(1);
var feeds = {};
feeds.sample_feed = require("./feeds/sample_feed_duplicates").feed;
@ -56,9 +56,10 @@ test('deduplicate feed from file', function(t) {
feeds : feeds,
indent : " ",
dedupe_fields : ["title", "guid"],
plugins_directories : [__dirname + '/../lib/example_plugins/']
plugins_directories : [__dirname + '/../examples/plugins/']
};
var rss_braider = RssBraider.createClient(braider_options);
rss_braider.logger.level('info');
rss_braider.processFeed('sample_feed', 'rss', function(err, data){
if (err) {
@ -69,7 +70,7 @@ test('deduplicate feed from file', function(t) {
});
});
test('sort by date desc', function(t) {
test('sort feed articles by date descending', function(t) {
t.plan(1);
var feeds = {};
feeds.sample_feed = require("./feeds/date_sort").feed;
@ -77,7 +78,7 @@ test('sort by date desc', function(t) {
feeds : feeds,
indent : " ",
date_sort_order : "desc",
plugins_directories : [__dirname + '/../lib/example_plugins/']
plugins_directories : [__dirname + '/../examples/plugins/']
};
var rss_braider = RssBraider.createClient(braider_options);
@ -90,7 +91,7 @@ test('sort by date desc', function(t) {
});
});
test('sort by date asc', function(t) {
test('sort feed articles by date ascending', function(t) {
t.plan(1);
var feeds = {};
feeds.sample_feed = require("./feeds/date_sort").feed;
@ -98,7 +99,7 @@ test('sort by date asc', function(t) {
feeds : feeds,
indent : " ",
date_sort_order : "asc",
plugins_directories : [__dirname + '/../lib/example_plugins/']
plugins_directories : [__dirname + '/../examples/plugins/']
};
var rss_braider = RssBraider.createClient(braider_options);
@ -111,4 +112,46 @@ test('sort by date asc', function(t) {
});
});
test('filter all articles out using plugin', function(t) {
t.plan(1);
var feeds = {};
feeds.sample_feed = require("./feeds/no_elements").feed;
var braider_options = {
feeds : feeds,
indent : " ",
date_sort_order : "asc",
plugins_directories : [__dirname + '/../examples/plugins/']
};
var rss_braider = RssBraider.createClient(braider_options);
rss_braider.logger.level('info');
rss_braider.processFeed('sample_feed', 'rss', function(err, data){
if (err) {
return t.fail(err);
}
// console.log(data);
t.equal(data, expectedOutput.emptyFeed);
});
});
test("Don't break when a filter fails and returns null", function(t) {
t.plan(1);
var feeds = {};
feeds.sample_feed = require("./feeds/sample_feed_bad_plugin").feed;
var braider_options = {
feeds : feeds,
indent : " ",
date_sort_order : "asc",
plugins_directories : [__dirname + '/../examples/plugins/']
};
var rss_braider = RssBraider.createClient(braider_options);
rss_braider.logger.level('info');
rss_braider.processFeed('sample_feed', 'rss', function(err, data){
if (err) {
return t.fail(err);
}
// console.log(data);
t.equal(data, expectedOutput.fileFeedBadPlugin);
});
});