Compare commits

..

No commits in common. "master" and "release" have entirely different histories.

31 changed files with 198 additions and 352 deletions

2
.gitignore vendored
View File

@ -1,2 +0,0 @@
node_modules
package-lock.json

View File

@ -1,5 +1,7 @@
language: node_js language: node_js
node_js: node_js:
- "0.12" - "0.12"
- "8.11.3" - "0.11"
- "10.5.0" - "0.10"
- iojs
- iojs-v1.0.2

View File

@ -1,5 +1,4 @@
[![Build Status](https://travis-ci.org/rv-kip/rss-braider.svg?branch=master)](https://travis-ci.org/rv-kip/rss-braider) [![Build Status](https://travis-ci.org/KQED/rss-braider.svg?branch=master)](https://travis-ci.org/KQED/rss-braider)
[![dependencies Status](https://david-dm.org/rv-kip/rss-braider/status.svg)](https://david-dm.org/rv-kip/rss-braider)
## Summary ## Summary
Braid/aggregate one or more RSS feeds (file or url) into a single feed (RSS or JSON output). Process resulting feed through specified plugins. Automatic deduplication Braid/aggregate one or more RSS feeds (file or url) into a single feed (RSS or JSON output). Process resulting feed through specified plugins. Automatic deduplication
@ -9,18 +8,15 @@ Braid/aggregate one or more RSS feeds (file or url) into a single feed (RSS or J
npm install rss-braider npm install rss-braider
``` ```
## Test ## Test
``` `npm test`
npm install
npm test
```
## Examples ## Examples
``` ```
$ cd examples $ cd examples
$ node simple.js (combines 3 sources) $ node simple.js (combines 3 sources)
$ node use_plugins.js (combines 3 sources and runs a transformation plugin) $ node plugins.js (combines 3 sources and runs a transformation plugin)
``` ```
### Code Example ## Code Example
```js ```js
var RssBraider = require('rss-braider'), var RssBraider = require('rss-braider'),
feeds = {}; feeds = {};
@ -52,14 +48,10 @@ feeds.simple_test_feed = {
var braider_options = { var braider_options = {
feeds : feeds, feeds : feeds,
indent : " ", indent : " ",
date_sort_order : "desc", // Newest first date_sort_order : "desc" // Newest first
log_level : "debug"
}; };
var rss_braider = RssBraider.createClient(braider_options); var rss_braider = RssBraider.createClient(braider_options);
// Override logging level (debug, info, warn, err, off)
rss_braider.logger.level('off');
// Output braided feed as rss. use 'json' for JSON output. // Output braided feed as rss. use 'json' for JSON output.
rss_braider.processFeed('simple_test_feed', 'rss', function(err, data){ rss_braider.processFeed('simple_test_feed', 'rss', function(err, data){
if (err) { if (err) {
@ -68,39 +60,3 @@ rss_braider.processFeed('simple_test_feed', 'rss', function(err, data){
console.log(data); console.log(data);
}); });
``` ```
## Plugins
Plugins provide custom manipulation and filtering of RSS items/articles. See `examples/plugins` for examples.
A plugin operates by modifying the `itemOptions` object or by returning `null` which will exclude the `item` (article) from the resulting feed (See `examples/plugins/filter_out_all_articles.js`).
The `itemsOptions` object gets passed to `node-rss` to generate the RSS feeds, so read the documentation on that module and its use of custom namespaces. (https://github.com/dylang/node-rss)
### Plugin Example
This plugin will capitalize the article title for all articles
```js
module.exports = function (item, itemOptions, source) {
if (!item || !itemOptions) {
return;
}
if (itemOptions.title) {
itemOptions.title = itemOptions.title.toUpperCase();
}
return itemOptions;
};
```
The plugin is registered with the feed in the feed config .js file and are run in order.
```js
var feed = {
"feed_name" : "feed with plugins",
"default_count" : 1,
"plugins" : ['capitalize_title', 'plugin_template'],
...
```
## Release Notes
### 1.0.0
Changed plugin architecture to allow filtering out of article/items by returning `-1` instead of a modified `itemsOptions` object. This is a breaking change as it will require existing plugins to return `itemsOptions` instead of modifying the reference. See `examples/plugins`.

View File

@ -2,7 +2,7 @@ var feed = {
"feed_name" : "feed with plugins", "feed_name" : "feed with plugins",
"default_count" : 1, "default_count" : 1,
"no_cdata_fields" : [], "no_cdata_fields" : [],
"plugins" : ['capitalize_title', 'plugin_template'], "plugins" : ['content_encoded'],
"meta" : { "meta" : {
"title": "NPR Braided Feed", "title": "NPR Braided Feed",
"description": "This is a test of two NPR sources from file. Plugins are applied." "description": "This is a test of two NPR sources from file. Plugins are applied."

View File

@ -6,8 +6,7 @@ feed_obj.filefeed = require("./config/feed_with_plugins").feed;
var braider_options = { var braider_options = {
feeds : feed_obj, feeds : feed_obj,
indent : " ", indent : " ",
plugins_directories : [__dirname + "/plugins/"], plugins_directories : [__dirname + "/plugins/"]
log_level : 'debug'
}; };
var rss_braider = RssBraider.createClient(braider_options); var rss_braider = RssBraider.createClient(braider_options);

View File

@ -1,8 +0,0 @@
module.exports = function (item, itemOptions, source) {
// This is an intentionally broken plugin for testing.
// This doesn't do anything and shouldn't be a template
// for other plugins
return;
};

View File

@ -1,7 +0,0 @@
module.exports = function (item, itemOptions, source) {
if (itemOptions.title) {
itemOptions.title = itemOptions.title.toUpperCase();
}
return itemOptions;
};

View File

@ -4,6 +4,9 @@
// <![CDATA[<p>Stewart let the news slip during a taping of his show today.]]> // <![CDATA[<p>Stewart let the news slip during a taping of his show today.]]>
// </content:encoded> // </content:encoded>
module.exports = function (item, itemOptions, source) { module.exports = function (item, itemOptions, source) {
if (!item || !itemOptions) {
return;
}
if (item["content:encoded"] && item["content:encoded"]["#"]){ if (item["content:encoded"] && item["content:encoded"]["#"]){
var content_encoded = item["content:encoded"]["#"]; var content_encoded = item["content:encoded"]["#"];
itemOptions.custom_elements.push( itemOptions.custom_elements.push(
@ -14,5 +17,4 @@ module.exports = function (item, itemOptions, source) {
} }
); );
} }
return itemOptions;
}; };

View File

@ -1,8 +0,0 @@
module.exports = function (item, itemOptions, source) {
if (!item || !itemOptions) {
return;
}
// This plugin removes all items by returning -1 instead of the processed itemOptions
return -1;
};

View File

@ -1,6 +0,0 @@
module.exports = function (item, itemOptions, source) {
// This plugin does no processing
// It's just a template
return itemOptions;
};

View File

@ -28,14 +28,10 @@ feeds.simple_test_feed = {
var braider_options = { var braider_options = {
feeds : feeds, feeds : feeds,
indent : " ", indent : " ",
date_sort_order : "desc", // Newest first date_sort_order : "desc" // Newest first
log_level : 'debug'
}; };
var rss_braider = RssBraider.createClient(braider_options); var rss_braider = RssBraider.createClient(braider_options);
// Set logging level (debug, info, warn, err)
rss_braider.logger.level('info');
rss_braider.processFeed('simple_test_feed', 'rss', function(err, data){ rss_braider.processFeed('simple_test_feed', 'rss', function(err, data){
if (err) { if (err) {
return console.log(err); return console.log(err);

View File

@ -1,32 +1,26 @@
// process feed-reader item into node-rss item // process feed-reader item into node-rss item
var FeedParser = require('feedparser'), var FeedParser = require('feedparser'),
bunyan = require('bunyan'), bunyan = require('bunyan'),
_ = require('lodash'), _ = require('lodash'),
async = require('async'), async = require('async'),
request = require('request'), request = require('request'),
RSS = require('rss'), RSS = require('rss'),
fs = require('fs'), fs = require('fs');
package_json = require('../package.json'),
logger;
var logger;
var RssBraider = function (options) { var RssBraider = function (options) {
if (!options) {
options = {};
}
this.feeds = options.feeds || null; this.feeds = options.feeds || null;
this.logger = options.logger || bunyan.createLogger({name: package_json.name}); this.logger = logger = options.logger || bunyan.createLogger({name: 'rss-braider'});
if (options.log_level) {
this.logger.level(options.log_level);
}
this.indent = options.indent || " "; this.indent = options.indent || " ";
this.dedupe_fields = options.dedupe_fields || []; // The fields to use to identify duplicate articles this.dedupe_fields = options.dedupe_fields || []; // The fields to use to identify duplicate articles
this.date_sort_order = options.date_sort_order || "desc"; this.date_sort_order = options.date_sort_order || "desc";
this.plugins_directories = options.plugins_directories || []; this.plugins_directories = options.plugins_directories || [];
// load plugins from plugins folder
// TODO, specify plugins location
this.plugins = {}; this.plugins = {};
this.loadPlugins(); this.loadPlugins();
}; };
// loadup self.plugins with the plugin functions // loadup self.plugins with the plugin functions
@ -34,7 +28,7 @@ RssBraider.prototype.loadPlugins = function () {
var self = this; var self = this;
if (self.plugins_directories.length < 1) { if (self.plugins_directories.length < 1) {
self.logger.debug("No plugins_directories specified. No plugins loaded."); // logger.info("No plugins_directories specified. No plugins loaded.");
} }
self.plugins_directories.forEach(function(path){ self.plugins_directories.forEach(function(path){
// load up each file and assign it to the plugins // load up each file and assign it to the plugins
@ -42,10 +36,10 @@ RssBraider.prototype.loadPlugins = function () {
filenames.forEach(function(filename){ filenames.forEach(function(filename){
var plugin_name = filename.replace(/.js$/, ''); var plugin_name = filename.replace(/.js$/, '');
if (self.plugins[plugin_name]) { if (self.plugins[plugin_name]) {
self.logger.warn("Duplicate plugin name: ", plugin_name, "Overwriting with newer plugin"); logger.warn("Duplicate plugin name: ", plugin_name, "Overwriting with newer plugin");
} }
self.plugins[plugin_name] = require(path + '/' + plugin_name); self.plugins[plugin_name] = require(path + '/' + plugin_name);
self.logger.debug("plugin registered:", plugin_name); // logger.info("plugin registered:", plugin_name);
}); });
}); });
}; };
@ -62,32 +56,35 @@ RssBraider.prototype.feedExists = function (feed_name) {
// trim down to desired count, dedupe and sort // trim down to desired count, dedupe and sort
RssBraider.prototype.processFeed = function(feed_name, format, callback) RssBraider.prototype.processFeed = function(feed_name, format, callback)
{ {
if (!format) {
format = 'rss';
}
var self = this, var self = this,
feed = self.feeds[feed_name], feed = self.feeds[feed_name],
feed_articles = []; feed_articles = [];
if (!format) { // logger.info("DEBUG processFeed: feed is set to " + feed_name);
format = 'rss';
}
if (!feed || !feed.sources || feed.sources.length < 1) { if (!feed || !feed.sources || feed.sources.length < 1) {
return callback("No definition for feed name: " + feed_name); return callback("No definition for feed name: " + feed_name);
} }
// Process each feed source through Feedparser to get articles.
// Then process each item/article through rss-braider and any plugins
async.each(feed.sources, function(source, callback) { async.each(feed.sources, function(source, callback) {
var count = source.count || feed.default_count || 10, // Number of articles per source var count = source.count || feed.default_count || 10, // Number of articles
url = source.feed_url || null, url = source.feed_url || null,
file_path = source.file_path || null, file_path = source.file_path || null,
source_articles = []; source_articles = [];
logger.debug("Requesting source:" + source.name + " at " + url + " for feed:" + feed_name);
// todo: Check if source.file is set and set up a fs stream read
var feedparser = new FeedParser(); var feedparser = new FeedParser();
if (url) { if (url) {
var req = request(url); var req = request(url);
// logger.info("request to", url);
req.on('error', function (error) { req.on('error', function (error) {
self.logger.error(error); logger.error(error);
}); });
req.on('response', function (res) { req.on('response', function (res) {
@ -102,27 +99,26 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
var filestream = fs.createReadStream(file_path); var filestream = fs.createReadStream(file_path);
filestream.pipe(feedparser); filestream.pipe(feedparser);
} else { } else {
self.logger.error("url or file_path not defined for feed: " + source.name); logger.error("url or file_path not defined for feed: " + source.name);
return callback(); return callback();
} }
feedparser.on('error', function(error) { feedparser.on('error', function(error) {
self.logger.error("feedparser",", source.name:", source.name, ", url:", source.feed_url, error.stack); logger.error("feedparser error:", error, "name:", source.name, "source:", source.feed_url);
}); });
// Collect the articles from this source // Collect the articles from this source
feedparser.on('readable', function() { feedparser.on('readable', function() {
// This is where the action is!
var stream = this, var stream = this,
item; item;
while ( !!(item = stream.read()) ) { while ( item = stream.read() ) {
if (source.feed_url) { if (source.feed_url) {
item.source_url = source.feed_url; item.source_url = source.feed_url;
} }
// Process Item/Article // Process Item/Article
var article = self.processItem(item, source, feed_name); var article = self.processItem(item, source, feed_name);
// plugins may filter items and return null
if (article) { if (article) {
source_articles.push(article); source_articles.push(article);
} }
@ -130,7 +126,7 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
}); });
feedparser.on("end", function(){ feedparser.on("end", function(){
// de-dupe , date sort, and trim this feed's articles and push them into array // sort and de-dupe this feed's articles and push them into array
source_articles = self.dedupe(source_articles, self.dedupe_fields); source_articles = self.dedupe(source_articles, self.dedupe_fields);
source_articles = self.date_sort(source_articles); source_articles = self.date_sort(source_articles);
source_articles = source_articles.slice(0, count); source_articles = source_articles.slice(0, count);
@ -140,14 +136,14 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
}, },
function(err){ function(err){
if (err) { if (err) {
self.logger.error(err); logger.error(err);
return callback(err); return callback(err);
} else { } else {
// Final Dedupe step and resort // Final Dedupe step and resort
feed_articles = self.dedupe(feed_articles, self.dedupe_fields); feed_articles = self.dedupe(feed_articles, self.dedupe_fields);
feed_articles = self.date_sort(feed_articles); feed_articles = self.date_sort(feed_articles);
// Create new feed with these articles. Follows node-rss spec // Create new feed with these articles
var options = { var options = {
title : feed.meta.title, title : feed.meta.title,
description : feed.meta.description, description : feed.meta.description,
@ -159,7 +155,6 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
copyright : feed.meta.copyright || null, copyright : feed.meta.copyright || null,
categories : feed.meta.categories || null, categories : feed.meta.categories || null,
custom_namespaces : feed.custom_namespaces || [], custom_namespaces : feed.custom_namespaces || [],
custom_elements : feed.meta.custom_elements || [],
no_cdata_fields : feed.no_cdata_fields no_cdata_fields : feed.no_cdata_fields
}; };
@ -171,10 +166,11 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
ret_string = JSON.stringify(newfeed); ret_string = JSON.stringify(newfeed);
break; break;
case 'rss': case 'rss':
case 'xml':
ret_string = newfeed.xml(self.indent); ret_string = newfeed.xml(self.indent);
break; break;
default: default:
self.logger.error("Unknown format:", format); logger.error("Unknown format:", format);
ret_string = "{}"; ret_string = "{}";
} }
@ -187,8 +183,8 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
RssBraider.prototype.processItem = function (item, source, feed_name) { RssBraider.prototype.processItem = function (item, source, feed_name) {
var self = this; var self = this;
if (!item || !source || !feed_name) { if (!item) {
self.logger.error("processItem: missing item, source, and/or feed_name"); logger.error("processItem: no item passed in");
return null; return null;
} }
// Basics // Basics
@ -205,46 +201,26 @@ RssBraider.prototype.processItem = function (item, source, feed_name) {
}; };
// Run the plugins specified by the "plugins" section of the // Run the plugins specified by the "plugins" section of the
// feed .js file to build out any custom elements or // feed config file to build out any custom elements or
// do transforms/filters // do transforms
var filteredItemOptions = self.runPlugins(item, itemOptions, source, feed_name); self.runPlugins(item, itemOptions, source, feed_name);
return filteredItemOptions; return itemOptions;
}; };
RssBraider.prototype.runPlugins = function (item, itemOptions, source, feed_name) { RssBraider.prototype.runPlugins = function (item, itemOptions, source, feed_name) {
var self = this, var self = this,
feed = self.feeds[feed_name] || {}, feed = self.feeds[feed_name] || {},
plugins_list = feed.plugins || [], plugins_list = feed.plugins || [];
ret_val,
filteredItemOptions;
// Process the item through the desired feed plugins // Process the item through the desired feed plugins
// plugins_list.forEach(function(plugin_name){ plugins_list.forEach(function(plugin_name){
for (var i = 0; i < plugins_list.length; i++) {
var plugin_name = plugins_list[i];
if (self.plugins[plugin_name]) { if (self.plugins[plugin_name]) {
filteredItemOptions = self.plugins[plugin_name](item, itemOptions, source); // logger.info("DEBUG runPlugins running " + plugin_name + " for item " + item.guid + " in feed: " + feed.meta.title);
self.plugins[plugin_name](item, itemOptions, source);
} else { } else {
self.logger.error("A plugin named '" + plugin_name + "' hasn't been registered"); logger.error("A plugin named '" + plugin_name + "' hasn't been registered");
} }
});
// A plugin returning -1 means skip this item
if (filteredItemOptions === -1) {
self.logger.debug("Plugin '" + plugin_name + "' filtered item from feed '" + feed.meta.title + "'", item.guid);
itemOptions = null;
break;
}
// Check that the plugin didn't just return null or undef, which would be bad.
if (!filteredItemOptions) {
self.logger.debug("Plugin '" + plugin_name + "' failed to return itemOptions for feed:'" + feed.meta.title + "'", item.guid);
filteredItemOptions = itemOptions; // Reset
}
// Prepare for next plugin.
itemOptions = filteredItemOptions;
}
return itemOptions;
}; };
// Dedupe articles in node-rss itemOptions format // Dedupe articles in node-rss itemOptions format
@ -252,7 +228,6 @@ RssBraider.prototype.runPlugins = function (item, itemOptions, source, feed_name
// operation on the articles array // operation on the articles array
// TODO, make this a plugin? // TODO, make this a plugin?
RssBraider.prototype.dedupe = function(articles_arr, fields){ RssBraider.prototype.dedupe = function(articles_arr, fields){
var self = this;
if ( !fields || fields.length < 1 ) { if ( !fields || fields.length < 1 ) {
return _.uniq(articles_arr); return _.uniq(articles_arr);
} else { } else {
@ -274,9 +249,8 @@ RssBraider.prototype.dedupe = function(articles_arr, fields){
// it's unique // it's unique
deduped_articles.push(article); deduped_articles.push(article);
} else { } else {
// The article matched all of another article's "dedupe" fields // The article matched all of another article's fields
// so filter it out (i.e. do nothing) // Do nothing
self.logger.debug("skipping duplicate", '"' + article.title + '"', article.guid);
} }
}); });
return deduped_articles; return deduped_articles;

View File

@ -6,6 +6,10 @@
// 'media:thumbnail' // 'media:thumbnail'
var _ = require('lodash'); var _ = require('lodash');
module.exports = function (item, itemOptions, source) { module.exports = function (item, itemOptions, source) {
if (!item || !itemOptions) {
return;
}
var thumbnail; var thumbnail;
if (item['media:thumbnail'] && item['media:thumbnail']['#']) { if (item['media:thumbnail'] && item['media:thumbnail']['#']) {
thumbnail = { thumbnail = {
@ -47,5 +51,4 @@ module.exports = function (item, itemOptions, source) {
} }
} }
} }
return itemOptions;
}; };

View File

@ -0,0 +1,20 @@
// Put the description into content:encoded block
// Ex:
// <content:encoded>
// <![CDATA[<p>Stewart let the news slip during a taping of his show today.]]>
// </content:encoded>
module.exports = function (item, itemOptions, source) {
if (!item || !itemOptions) {
return;
}
if (item["content:encoded"] && item["content:encoded"]["#"]){
var content_encoded = item["content:encoded"]["#"];
itemOptions.custom_elements.push(
{ "content:encoded":
{
_cdata: content_encoded
}
}
);
}
};

View File

@ -1,5 +1,8 @@
// define kqed source // define kqed source
module.exports = function (item, itemOptions, source) { module.exports = function (item, itemOptions, source) {
if (!item || !itemOptions || !source) {
return;
}
// Look for kqed namespace elements in source and add as custom elements for item // Look for kqed namespace elements in source and add as custom elements for item
// Ex: // Ex:
// <kqed:fullname>The California Report</kqed:fullname> // <kqed:fullname>The California Report</kqed:fullname>
@ -33,5 +36,4 @@ module.exports = function (item, itemOptions, source) {
{ 'kqed:feed_url': item.feed_url } { 'kqed:feed_url': item.feed_url }
); );
} }
return itemOptions;
}; };

View File

@ -1,5 +1,27 @@
// Pass through itunes content... module.exports = function (item, itemOptions, source) {
// if (!item || !itemOptions) {
return;
}
// 'itunes:summary':
// { '@': {},
// '#': 'Let KQED Arts help you find the best things to see, do, and explore in San Francisco, Oakland, San Jose, and the surrounding areas.' },
// 'itunes:author': { '@': {}, '#': 'KQED Arts' },
// 'itunes:explicit': { '@': {}, '#': 'no' },
// 'itunes:image': { '@': [Object] },
// 'itunes:owner': { '@': {}, 'itunes:name': [Object], 'itunes:email': [Object] },
// 'rss:managingeditor':
// { '@': {},
// '#': 'ondemand@kqed.org (KQED Arts)',
// name: 'KQED Arts',
// email: 'ondemand@kqed.org' },
// 'rss:copyright':
// { '@': {},
// '#': 'Copyright © 2015 KQED Inc. All Rights Reserved.' },
// 'itunes:subtitle': { '@': {}, '#': 'KQED Public Media for Northern CA' },
// 'rss:image': { '@': {}, title: [Object], url: [Object], link: [Object] },
// 'itunes:category': [ [Object], [Object], [Object] ],
// <itunes:summary> // <itunes:summary>
// Let KQED Arts help you find the best things to see, do, and explore in San Francisco, Oakland, San Jose, and the surrounding areas. // Let KQED Arts help you find the best things to see, do, and explore in San Francisco, Oakland, San Jose, and the surrounding areas.
// </itunes:summary> // </itunes:summary>
@ -11,9 +33,28 @@
// <itunes:email>ondemand@kqed.org</itunes:email> // <itunes:email>ondemand@kqed.org</itunes:email>
// </itunes:owner> // </itunes:owner>
// <managingEditor>ondemand@kqed.org (KQED Arts)</managingEditor> // <managingEditor>ondemand@kqed.org (KQED Arts)</managingEditor>
// <copyright>Copyright © 2015 KQED Inc. All Rights Reserved.</copyright>
// <itunes:subtitle>KQED Public Media for Northern CA</itunes:subtitle> // <itunes:subtitle>KQED Public Media for Northern CA</itunes:subtitle>
module.exports = function (item, itemOptions, source) { // *link
// *comments
// pubDate
// *dc:creator
// *category
// guid
// description
// *content:encoded
// *wfw:commentRss
// *slash:comments
// enclosure
// itunes:subtitle
// itunes:summary
// itunes:author
// itunes:explicit
// *media:content (multiple)
// *media:thumbnail
// Pass through itunes content
var pass_through_arr = ['itunes:summary', 'itunes:author', 'itunes:explicit', ]; var pass_through_arr = ['itunes:summary', 'itunes:author', 'itunes:explicit', ];
pass_through_arr.forEach(function(element){ pass_through_arr.forEach(function(element){
if (item[element] && item[element]['#']) { if (item[element] && item[element]['#']) {
@ -41,5 +82,4 @@ module.exports = function (item, itemOptions, source) {
} }
}); });
} }
return itemOptions;
}; };

View File

@ -1,4 +1,7 @@
module.exports = function (item, itemOptions, source) { module.exports = function (item, itemOptions, source) {
if (!item || !itemOptions) {
return;
}
// wfw // wfw
if (item["wfw:commentrss"] && item["wfw:commentrss"]["#"]){ if (item["wfw:commentrss"] && item["wfw:commentrss"]["#"]){
itemOptions.custom_elements.push({ "wfw:commentRss": item["wfw:commentrss"]["#"]}); itemOptions.custom_elements.push({ "wfw:commentRss": item["wfw:commentrss"]["#"]});
@ -8,5 +11,4 @@ module.exports = function (item, itemOptions, source) {
if (item["slash:comments"] && item["slash:comments"]["#"]){ if (item["slash:comments"] && item["slash:comments"]["#"]){
itemOptions.custom_elements.push({ "slash:comments": item["slash:comments"]["#"]}); itemOptions.custom_elements.push({ "slash:comments": item["slash:comments"]["#"]});
} }
return itemOptions;
}; };

View File

@ -1,9 +1,9 @@
{ {
"name": "rss-braider", "name": "rss-braider",
"version": "1.2.4", "version": "0.1.4",
"description": "Braid/aggregate/combine RSS feeds into a single RSS (or JSON) document. Optionally process through specified plugins.", "description": "Braid/aggregate RSS feeds into a single RSS (or JSON) document. Process resulting feed through specified plugins.",
"main": "index.js", "main": "index.js",
"repository": "https://github.com/rv-kip/rss-braider", "repository": "https://github.com/KQED/rss-braider",
"scripts": { "scripts": {
"test": "tape test" "test": "tape test"
}, },
@ -11,37 +11,33 @@
"rss", "rss",
"braider", "braider",
"combiner", "combiner",
"aggregator", "rss",
"json", "json",
"xml",
"feed", "feed",
"feeds", "feeds",
"feed builder",
"syndicate", "syndicate",
"syndication", "syndication",
"wordpress", "wordpress",
"blogs", "blogs",
"distribution", "distribution"
"podcasts",
"atom"
], ],
"bugs": { "bugs": {
"url": "http://github.com/rv-kip/rss-braider/issues", "url": "http://github.com/KQED/rss-braider/issues",
"email": "kgebhardt23@gmail.com" "email": "kgebhardt@kqed.org"
}, },
"author": "Kip Gebhardt <kgebhardt23@gmail.com>", "author": "Kip Gebhardt <kgebhardt@kqed.org>",
"license": "MIT", "license": "MIT",
"dependencies": { "dependencies": {
"async": "^2.6.1", "async": "^1.0.0",
"bunyan": "^1.8.12", "bunyan": "^1.3.5",
"feedparser": "^2.2.4", "feedparser": "^1.1.1",
"include-folder": "^1.0.0", "include-folder": "^0.9.0",
"lodash": "^4.17.10", "lodash": "^3.9.3",
"request": "^2.87.0", "request": "^2.56.0",
"rss": "^1.2.2" "rss": "git://github.com/rv-kip/node-rss.git#8d1420"
}, },
"devDependencies": { "devDependencies": {
"mockdate": "^2.0.1", "tape": "^4.0.0",
"tape": "^4.9.1" "mockdate": "^1.0.3"
} }
} }

View File

@ -1,14 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?><rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:atom="http://www.w3.org/2005/Atom" version="2.0" xmlns:slash="http://purl.org/rss/1.0/modules/slash/" xmlns:media="http://search.yahoo.com/mrss/" xmlns:ev="http://purl.org/rss/2.0/modules/event/" xmlns:sy="http://purl.org/rss/1.0/modules/syndication/" xmlns:wfw="http://wellformedweb.org/CommentAPI/"> <?xml version="1.0" encoding="UTF-8"?>
<rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:atom="http://www.w3.org/2005/Atom" version="2.0" xmlns:slash="http://purl.org/rss/1.0/modules/slash/" xmlns:media="http://search.yahoo.com/mrss/" xmlns:ev="http://purl.org/rss/2.0/modules/event/" xmlns:sy="http://purl.org/rss/1.0/modules/syndication/" xmlns:wfw="http://wellformedweb.org/CommentAPI/">
<channel> <channel>
<title><![CDATA[Test File Feed]]></title> <title><![CDATA[Test File Feed]]></title>
<description><![CDATA[This feed comes from a file]]></description> <description>This feed comes from a file</description>
<link>http://github.com/dylang/node-rss</link> <link>http://github.com/dylang/node-rss</link>
<generator>rss-braider</generator> <generator>rss-braider</generator>
<lastBuildDate>Wed, 31 Dec 2014 00:00:01 GMT</lastBuildDate> <lastBuildDate>Wed, 31 Dec 2014 00:00:01 GMT</lastBuildDate>
<item> <item>
<title><![CDATA[Rent Hike For Dance Mission Theater Has Artists Worried About Uncertain Future]]></title> <title><![CDATA[Rent Hike For Dance Mission Theater Has Artists Worried About Uncertain Future]]></title>
<description><![CDATA[<p>Stepping out of BART at 24th and Mission at most hours of the day, one is likely to hear the pulse of African drums, hip-hop or salsa emanating from the second-floor studios of Dance Brigade's Dance Mission Theater. But that music may not continue forever.</p> <description>&lt;p&gt;Stepping out of BART at 24th and Mission at most hours of the day, one is likely to hear the pulse of African drums, hip-hop or salsa emanating from the second-floor studios of Dance Brigade&apos;s Dance Mission Theater. But that music may not continue forever.&lt;/p&gt;
<p>The performance space and dance school <a href="http://ww2.kqed.org/news/2014/12/20/dance-mission-theater-rent-increase-worries-artists/" target="_self" id="rssmi_more"> ...read more</a>]]></description> &lt;p&gt;The performance space and dance school &lt;a href=&quot;http://ww2.kqed.org/news/2014/12/20/dance-mission-theater-rent-increase-worries-artists/&quot; target=&quot;_self&quot; id=&quot;rssmi_more&quot;&gt; ...read more&lt;/a&gt;</description>
<link>http://ww2.kqed.org/news/2014/12/20/dance-mission-theater-rent-increase-worries-artists/</link> <link>http://ww2.kqed.org/news/2014/12/20/dance-mission-theater-rent-increase-worries-artists/</link>
<guid isPermaLink="false">http://ww2.kqed.org/arts/2014/12/20/rent-hike-for-dance-mission-theater-has-artists-worried-about-uncertain-future/</guid> <guid isPermaLink="false">http://ww2.kqed.org/arts/2014/12/20/rent-hike-for-dance-mission-theater-has-artists-worried-about-uncertain-future/</guid>
<dc:creator><![CDATA[KQED Arts]]></dc:creator> <dc:creator><![CDATA[KQED Arts]]></dc:creator>
@ -26,7 +27,7 @@
</item> </item>
<item> <item>
<title><![CDATA[Bob Miller: Teasing Science Lessons from Everyday Phenomena]]></title> <title><![CDATA[Bob Miller: Teasing Science Lessons from Everyday Phenomena]]></title>
<description><![CDATA[Until February 5, 2015, you can visit the main branch of the San Francisco Public Library and be momentarily transported through space and time to the early days of the Exploratorium via the life and work of Bob Miller, who died in 2007.]]></description> <description>Until February 5, 2015, you can visit the main branch of the San Francisco Public Library and be momentarily transported through space and time to the early days of the Exploratorium via the life and work of Bob Miller, who died in 2007.</description>
<link>http://ww2.kqed.org/arts/2014/12/20/bob-miller-teasing-science-lessons-from-everyday-phenomena/</link> <link>http://ww2.kqed.org/arts/2014/12/20/bob-miller-teasing-science-lessons-from-everyday-phenomena/</link>
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10220517</guid> <guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10220517</guid>
<dc:creator><![CDATA[Sarah Hotchkiss]]></dc:creator> <dc:creator><![CDATA[Sarah Hotchkiss]]></dc:creator>
@ -54,7 +55,7 @@
</item> </item>
<item> <item>
<title><![CDATA[Light Art Brings Holiday Glow to Darkest Nights]]></title> <title><![CDATA[Light Art Brings Holiday Glow to Darkest Nights]]></title>
<description><![CDATA[In the dark of winter, San Franciscans with an urge to celebrate the light can visit a new wealth of illuminated art installations. This video tour offers a preview of some of the more dazzling works.]]></description> <description>In the dark of winter, San Franciscans with an urge to celebrate the light can visit a new wealth of illuminated art installations. This video tour offers a preview of some of the more dazzling works.</description>
<link>http://ww2.kqed.org/arts/2014/12/21/on-darkest-nights-illuminated-art-brings-holiday-glow/</link> <link>http://ww2.kqed.org/arts/2014/12/21/on-darkest-nights-illuminated-art-brings-holiday-glow/</link>
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10231062</guid> <guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10231062</guid>
<dc:creator><![CDATA[KQED Arts]]></dc:creator> <dc:creator><![CDATA[KQED Arts]]></dc:creator>
@ -71,7 +72,7 @@
</item> </item>
<item> <item>
<title><![CDATA[93 Til Infinity: Watch Bay Area Musicians Remix Classic 90s Hip-Hop]]></title> <title><![CDATA[93 Til Infinity: Watch Bay Area Musicians Remix Classic 90s Hip-Hop]]></title>
<description><![CDATA[One night back in November, the YBCA took a trip to 1993 to celebrate a groundbreaking year for hip hop.]]></description> <description>One night back in November, the YBCA took a trip to 1993 to celebrate a groundbreaking year for hip hop.</description>
<link>http://ww2.kqed.org/arts/2014/12/22/93-til-infinity-watch-bay-area-musicians-remix-classic-90s-hip-hop/</link> <link>http://ww2.kqed.org/arts/2014/12/22/93-til-infinity-watch-bay-area-musicians-remix-classic-90s-hip-hop/</link>
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10230592</guid> <guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10230592</guid>
<dc:creator><![CDATA[Kevin L. Jones]]></dc:creator> <dc:creator><![CDATA[Kevin L. Jones]]></dc:creator>
@ -90,7 +91,7 @@
</item> </item>
<item> <item>
<title><![CDATA[Protest Icons: Not Just for Show]]></title> <title><![CDATA[Protest Icons: Not Just for Show]]></title>
<description><![CDATA[Hands up, umbrellas out, hoodies on, fists raised -- the icons of protest have long played a significant role in movements of social change and revolution. Some of the most potent protest icons of 2014 appeared in just the last four months of the year.]]></description> <description>Hands up, umbrellas out, hoodies on, fists raised -- the icons of protest have long played a significant role in movements of social change and revolution. Some of the most potent protest icons of 2014 appeared in just the last four months of the year.</description>
<link>http://ww2.kqed.org/arts/2014/12/23/protest-icons-not-just-for-show/</link> <link>http://ww2.kqed.org/arts/2014/12/23/protest-icons-not-just-for-show/</link>
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10189953</guid> <guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10189953</guid>
<dc:creator><![CDATA[Michele Carlson]]></dc:creator> <dc:creator><![CDATA[Michele Carlson]]></dc:creator>

View File

@ -1,13 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?><rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:atom="http://www.w3.org/2005/Atom" version="2.0" xmlns:slash="http://purl.org/rss/1.0/modules/slash/" xmlns:media="http://search.yahoo.com/mrss/" xmlns:ev="http://purl.org/rss/2.0/modules/event/" xmlns:sy="http://purl.org/rss/1.0/modules/syndication/" xmlns:wfw="http://wellformedweb.org/CommentAPI/"> <?xml version="1.0" encoding="UTF-8"?>
<rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:atom="http://www.w3.org/2005/Atom" version="2.0" xmlns:slash="http://purl.org/rss/1.0/modules/slash/" xmlns:media="http://search.yahoo.com/mrss/" xmlns:ev="http://purl.org/rss/2.0/modules/event/" xmlns:sy="http://purl.org/rss/1.0/modules/syndication/" xmlns:wfw="http://wellformedweb.org/CommentAPI/">
<channel> <channel>
<title><![CDATA[Test File Feed]]></title> <title><![CDATA[Test File Feed]]></title>
<description><![CDATA[This feed comes from a file]]></description> <description>This feed comes from a file</description>
<link>http://github.com/dylang/node-rss</link> <link>http://github.com/dylang/node-rss</link>
<generator>rss-braider</generator> <generator>rss-braider</generator>
<lastBuildDate>Wed, 31 Dec 2014 00:00:01 GMT</lastBuildDate> <lastBuildDate>Wed, 31 Dec 2014 00:00:01 GMT</lastBuildDate>
<item> <item>
<title><![CDATA[Top 10 Movie Moments of 2014]]></title> <title><![CDATA[Top 10 Movie Moments of 2014]]></title>
<description><![CDATA[A handful of scenes resonate from a mediocre year for film.]]></description> <description>A handful of scenes resonate from a mediocre year for film.</description>
<link>http://ww2.kqed.org/arts/2014/12/26/top-10-movie-moments-of-2014/</link> <link>http://ww2.kqed.org/arts/2014/12/26/top-10-movie-moments-of-2014/</link>
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10222283</guid> <guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10222283</guid>
<dc:creator><![CDATA[Michael Fox]]></dc:creator> <dc:creator><![CDATA[Michael Fox]]></dc:creator>
@ -55,7 +56,7 @@
</item> </item>
<item> <item>
<title><![CDATA[Radio Show: Rockin Roots to Radical Rauschenberg]]></title> <title><![CDATA[Radio Show: Rockin Roots to Radical Rauschenberg]]></title>
<description><![CDATA[Cy Musiker and David Wiegand share their picks for great events around the Bay Area this week.]]></description> <description>Cy Musiker and David Wiegand share their picks for great events around the Bay Area this week.</description>
<link>http://ww2.kqed.org/arts/2014/12/25/radio-show-rockin-roots-to-radical-rauschenberg/</link> <link>http://ww2.kqed.org/arts/2014/12/25/radio-show-rockin-roots-to-radical-rauschenberg/</link>
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10238611</guid> <guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10238611</guid>
<dc:creator><![CDATA[The Do List]]></dc:creator> <dc:creator><![CDATA[The Do List]]></dc:creator>
@ -97,7 +98,7 @@
</item> </item>
<item> <item>
<title><![CDATA[Watch a Hip Hop Orchestra Perform Wu-Tang Clans “C.R.E.A.M”]]></title> <title><![CDATA[Watch a Hip Hop Orchestra Perform Wu-Tang Clans “C.R.E.A.M”]]></title>
<description><![CDATA[Above: Ensemble Mik Nawooj (EMN), led by music director, composer/pianist JooWan Kim, reinterprets Wu-Tang Clans &#8220;C.R.E.A.M.&#8221; at YBCA&#8217;s Clas/Sick Hip Hop: 1993 Edition. Ensemble Mik Nawooj (EMN) is the brainchild of composer/pianist JooWan Kim, who introduces western-European classical techniques into hip-hop, rock, and pop. The group&#8217;s lineup includes traditional Pierrot ensemble instrumentation (flute, clarinet, violin, cello and piano) with [&#8230;]]]></description> <description>Above: Ensemble Mik Nawooj (EMN), led by music director, composer/pianist JooWan Kim, reinterprets Wu-Tang Clans &amp;#8220;C.R.E.A.M.&amp;#8221; at YBCA&amp;#8217;s Clas/Sick Hip Hop: 1993 Edition. Ensemble Mik Nawooj (EMN) is the brainchild of composer/pianist JooWan Kim, who introduces western-European classical techniques into hip-hop, rock, and pop. The group&amp;#8217;s lineup includes traditional Pierrot ensemble instrumentation (flute, clarinet, violin, cello and piano) with [&amp;#8230;]</description>
<link>http://ww2.kqed.org/arts/2014/12/24/a-hip-hop-orchestra-does-wu-tang-clans-c-r-e-a-m/</link> <link>http://ww2.kqed.org/arts/2014/12/24/a-hip-hop-orchestra-does-wu-tang-clans-c-r-e-a-m/</link>
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10238600</guid> <guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10238600</guid>
<dc:creator><![CDATA[Siouxsie Oki]]></dc:creator> <dc:creator><![CDATA[Siouxsie Oki]]></dc:creator>
@ -117,7 +118,7 @@
</item> </item>
<item> <item>
<title><![CDATA[Watch a Hip-Hop Funk Big Band Pay Homage to Saafirs “Light Sleeper”]]></title> <title><![CDATA[Watch a Hip-Hop Funk Big Band Pay Homage to Saafirs “Light Sleeper”]]></title>
<description><![CDATA[Above: Kev Choice Ensemble reinterprets Saafir&#8217;s &#8220;Light Sleeper&#8221; at YBCA&#8217;s Clas/Sick Hip Hop: 1993 Edition. Oakland-based artist Kev Choice is a pianist, MC, producer, bandleader, sideman, music historian and urban griot dedicated to cultural expression. Through his band Kev Choice Ensemble, he produces and performs his own material, including original jazz and classical compositions as well as classical, [&#8230;]]]></description> <description>Above: Kev Choice Ensemble reinterprets Saafir&amp;#8217;s &amp;#8220;Light Sleeper&amp;#8221; at YBCA&amp;#8217;s Clas/Sick Hip Hop: 1993 Edition. Oakland-based artist Kev Choice is a pianist, MC, producer, bandleader, sideman, music historian and urban griot dedicated to cultural expression. Through his band Kev Choice Ensemble, he produces and performs his own material, including original jazz and classical compositions as well as classical, [&amp;#8230;]</description>
<link>http://ww2.kqed.org/arts/2014/12/24/a-jazz-hip-hop-funk-big-band-pays-homage-to-saafirs-light-sleeper/</link> <link>http://ww2.kqed.org/arts/2014/12/24/a-jazz-hip-hop-funk-big-band-pays-homage-to-saafirs-light-sleeper/</link>
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10238610</guid> <guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10238610</guid>
<dc:creator><![CDATA[Siouxsie Oki]]></dc:creator> <dc:creator><![CDATA[Siouxsie Oki]]></dc:creator>
@ -138,7 +139,7 @@
</item> </item>
<item> <item>
<title><![CDATA[Jason McHenry and His Quest for One Million Paintings]]></title> <title><![CDATA[Jason McHenry and His Quest for One Million Paintings]]></title>
<description><![CDATA[Though the original intent was to make as much art as possible, the project somehow morphed into a goal of producing a seemingly preposterous number of paintings; now the mission of the <em>One Thousand Thousand</em> project is to amass that amount with the help of friends.]]></description> <description>Though the original intent was to make as much art as possible, the project somehow morphed into a goal of producing a seemingly preposterous number of paintings; now the mission of the &lt;em&gt;One Thousand Thousand&lt;/em&gt; project is to amass that amount with the help of friends.</description>
<link>http://ww2.kqed.org/arts/2014/12/24/jason-mchenry-and-his-quest-to-amass-one-million-paintings/</link> <link>http://ww2.kqed.org/arts/2014/12/24/jason-mchenry-and-his-quest-to-amass-one-million-paintings/</link>
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10218934</guid> <guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10218934</guid>
<dc:creator><![CDATA[Brian Eder]]></dc:creator> <dc:creator><![CDATA[Brian Eder]]></dc:creator>

View File

@ -1,9 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?><rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:atom="http://www.w3.org/2005/Atom" version="2.0">
<channel>
<title><![CDATA[A Feed with no elements]]></title>
<description><![CDATA[This feed will have no elements as a result of the filter_all_articles plugin. Used for unit tests.]]></description>
<link>http://github.com/dylang/node-rss</link>
<generator>rss-braider</generator>
<lastBuildDate>Wed, 31 Dec 2014 00:00:01 GMT</lastBuildDate>
</channel>
</rss>

View File

@ -1,34 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?><rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:atom="http://www.w3.org/2005/Atom" version="2.0">
<channel>
<title><![CDATA[A Feed with no elements]]></title>
<description><![CDATA[This feed will have no elements as a result of the filter_all_articles plugin. Used for unit tests.]]></description>
<link>http://github.com/dylang/node-rss</link>
<generator>rss-braider</generator>
<lastBuildDate>Wed, 31 Dec 2014 00:00:01 GMT</lastBuildDate>
<item>
<title><![CDATA[Rent Hike For Dance Mission Theater Has Artists Worried About Uncertain Future]]></title>
<description><![CDATA[<p>Stepping out of BART at 24th and Mission at most hours of the day, one is likely to hear the pulse of African drums, hip-hop or salsa emanating from the second-floor studios of Dance Brigade's Dance Mission Theater. But that music may not continue forever.</p>
<p>The performance space and dance school <a href="http://ww2.kqed.org/news/2014/12/20/dance-mission-theater-rent-increase-worries-artists/" target="_self" id="rssmi_more"> ...read more</a>]]></description>
<link>http://ww2.kqed.org/news/2014/12/20/dance-mission-theater-rent-increase-worries-artists/</link>
<guid isPermaLink="false">http://ww2.kqed.org/arts/2014/12/20/rent-hike-for-dance-mission-theater-has-artists-worried-about-uncertain-future/</guid>
<dc:creator><![CDATA[KQED Arts]]></dc:creator>
<pubDate>Sat, 20 Dec 2014 09:00:22 GMT</pubDate>
</item>
<item>
<title><![CDATA[Bob Miller: Teasing Science Lessons from Everyday Phenomena]]></title>
<description><![CDATA[Until February 5, 2015, you can visit the main branch of the San Francisco Public Library and be momentarily transported through space and time to the early days of the Exploratorium via the life and work of Bob Miller, who died in 2007.]]></description>
<link>http://ww2.kqed.org/arts/2014/12/20/bob-miller-teasing-science-lessons-from-everyday-phenomena/</link>
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10220517</guid>
<dc:creator><![CDATA[Sarah Hotchkiss]]></dc:creator>
<pubDate>Sat, 20 Dec 2014 14:00:47 GMT</pubDate>
</item>
<item>
<title><![CDATA[Light Art Brings Holiday Glow to Darkest Nights]]></title>
<description><![CDATA[In the dark of winter, San Franciscans with an urge to celebrate the light can visit a new wealth of illuminated art installations. This video tour offers a preview of some of the more dazzling works.]]></description>
<link>http://ww2.kqed.org/arts/2014/12/21/on-darkest-nights-illuminated-art-brings-holiday-glow/</link>
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10231062</guid>
<dc:creator><![CDATA[KQED Arts]]></dc:creator>
<pubDate>Sun, 21 Dec 2014 14:00:08 GMT</pubDate>
</item>
</channel>
</rss>

View File

@ -1,13 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?><rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:atom="http://www.w3.org/2005/Atom" version="2.0" xmlns:slash="http://purl.org/rss/1.0/modules/slash/" xmlns:media="http://search.yahoo.com/mrss/" xmlns:ev="http://purl.org/rss/2.0/modules/event/" xmlns:sy="http://purl.org/rss/1.0/modules/syndication/" xmlns:wfw="http://wellformedweb.org/CommentAPI/"> <?xml version="1.0" encoding="UTF-8"?>
<rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:atom="http://www.w3.org/2005/Atom" version="2.0" xmlns:slash="http://purl.org/rss/1.0/modules/slash/" xmlns:media="http://search.yahoo.com/mrss/" xmlns:ev="http://purl.org/rss/2.0/modules/event/" xmlns:sy="http://purl.org/rss/1.0/modules/syndication/" xmlns:wfw="http://wellformedweb.org/CommentAPI/">
<channel> <channel>
<title><![CDATA[Test File Feed]]></title> <title><![CDATA[Test File Feed]]></title>
<description><![CDATA[This feed comes from a file]]></description> <description>This feed comes from a file</description>
<link>http://github.com/dylang/node-rss</link> <link>http://github.com/dylang/node-rss</link>
<generator>rss-braider</generator> <generator>rss-braider</generator>
<lastBuildDate>Wed, 31 Dec 2014 00:00:01 GMT</lastBuildDate> <lastBuildDate>Wed, 31 Dec 2014 00:00:01 GMT</lastBuildDate>
<item> <item>
<title><![CDATA[Top 10 Movie Moments of 2014]]></title> <title><![CDATA[Top 10 Movie Moments of 2014]]></title>
<description><![CDATA[A handful of scenes resonate from a mediocre year for film.]]></description> <description>A handful of scenes resonate from a mediocre year for film.</description>
<link>http://ww2.kqed.org/arts/2014/12/26/top-10-movie-moments-of-2014/</link> <link>http://ww2.kqed.org/arts/2014/12/26/top-10-movie-moments-of-2014/</link>
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10222283</guid> <guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10222283</guid>
<dc:creator><![CDATA[Michael Fox]]></dc:creator> <dc:creator><![CDATA[Michael Fox]]></dc:creator>

View File

@ -1,13 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?><rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:atom="http://www.w3.org/2005/Atom" version="2.0" xmlns:slash="http://purl.org/rss/1.0/modules/slash/" xmlns:media="http://search.yahoo.com/mrss/" xmlns:ev="http://purl.org/rss/2.0/modules/event/" xmlns:sy="http://purl.org/rss/1.0/modules/syndication/" xmlns:wfw="http://wellformedweb.org/CommentAPI/"> <?xml version="1.0" encoding="UTF-8"?>
<rss xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:atom="http://www.w3.org/2005/Atom" version="2.0" xmlns:slash="http://purl.org/rss/1.0/modules/slash/" xmlns:media="http://search.yahoo.com/mrss/" xmlns:ev="http://purl.org/rss/2.0/modules/event/" xmlns:sy="http://purl.org/rss/1.0/modules/syndication/" xmlns:wfw="http://wellformedweb.org/CommentAPI/">
<channel> <channel>
<title><![CDATA[Test File Feed]]></title> <title><![CDATA[Test File Feed]]></title>
<description><![CDATA[This feed comes from a file]]></description> <description>This feed comes from a file</description>
<link>http://github.com/dylang/node-rss</link> <link>http://github.com/dylang/node-rss</link>
<generator>rss-braider</generator> <generator>rss-braider</generator>
<lastBuildDate>Wed, 31 Dec 2014 00:00:01 GMT</lastBuildDate> <lastBuildDate>Wed, 31 Dec 2014 00:00:01 GMT</lastBuildDate>
<item> <item>
<title><![CDATA[Top 10 Movie Moments of 2014]]></title> <title><![CDATA[Top 10 Movie Moments of 2014]]></title>
<description><![CDATA[A handful of scenes resonate from a mediocre year for film.]]></description> <description>A handful of scenes resonate from a mediocre year for film.</description>
<link>http://ww2.kqed.org/arts/2014/12/26/top-10-movie-moments-of-2014/</link> <link>http://ww2.kqed.org/arts/2014/12/26/top-10-movie-moments-of-2014/</link>
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10222283</guid> <guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10222283</guid>
<dc:creator><![CDATA[Michael Fox]]></dc:creator> <dc:creator><![CDATA[Michael Fox]]></dc:creator>

View File

@ -2,7 +2,7 @@ var feed = {
"feed_name" : "test file feed", "feed_name" : "test file feed",
"default_count" : 1, "default_count" : 1,
"no_cdata_fields" : ['description'], "no_cdata_fields" : ['description'],
"plugins" : ['kqed', 'add_content_encoded_block', 'wfw_slash_comments', 'add_media_thumbnail'], "plugins" : ['kqed', 'content_encoded', 'wfw_slash_comments', 'add_media_thumbnail'],
"meta" : { "meta" : {
"title": "Test File Feed", "title": "Test File Feed",
"description": "This feed comes from a file", "description": "This feed comes from a file",

View File

@ -1,18 +0,0 @@
var feed = {
"feed_name" : "no_elements",
"plugins" : ['filter_out_all_articles'], // No articles make it through
"meta" : {
"title" : "A Feed with no elements",
"description" : "This feed will have no elements as a result of the filter_all_articles plugin. Used for unit tests.",
"url" : "http://rss.nytimes.com/services/xml/rss/nyt/Technology.xml",
},
"sources" : [
{
"name" : "nyt_tech",
"feed_url" : "http://rss.nytimes.com/services/xml/rss/nyt/Technology.xml",
"count" : 5,
"fullname" : "NYT Technology"
}
]
};
exports.feed = feed;

View File

@ -16,10 +16,11 @@ var feed = {
}, },
"sources" : [ "sources" : [
{ {
"name" : "sample_source", "name" : "sample_feed",
"count" : 1, "count" : 1,
"file_path" : __dirname + "/../input_files/sample_feed.xml", "file_path" : __dirname + "/../input_files/sample_feed.xml",
} },
] ]
}; };
exports.feed = feed; exports.feed = feed;

View File

@ -1,17 +0,0 @@
var feed = {
"feed_name" : "no_elements",
// "plugins" : ['bad_plugin'], // Intentionally bad plugin for testing
"meta" : {
"title" : "A Feed with no elements",
"description" : "This feed will have no elements as a result of the filter_all_articles plugin. Used for unit tests.",
"url" : "http://rss.nytimes.com/services/xml/rss/nyt/Technology.xml",
},
"sources" : [
{
"name" : "sample_source",
"count" : 3,
"file_path" : __dirname + "/../input_files/sample_feed.xml",
}
]
};
exports.feed = feed;

View File

@ -2,7 +2,7 @@ var feed = {
"feed_name" : "test file feed", "feed_name" : "test file feed",
"default_count" : 1, "default_count" : 1,
"no_cdata_fields" : ['description'], "no_cdata_fields" : ['description'],
"plugins" : ['kqed', 'add_content_encoded_block', 'wfw_slash_comments', 'add_media_thumbnail'], "plugins" : ['kqed', 'content_encoded', 'wfw_slash_comments', 'add_media_thumbnail'],
"meta" : { "meta" : {
"title": "Test File Feed", "title": "Test File Feed",
"description": "This feed comes from a file", "description": "This feed comes from a file",

View File

@ -2,7 +2,7 @@ var feed = {
"feed_name" : "test file feed", "feed_name" : "test file feed",
"default_count" : 1, "default_count" : 1,
"no_cdata_fields" : ['description'], "no_cdata_fields" : ['description'],
"plugins" : ['kqed', 'add_content_encoded_block', 'wfw_slash_comments', 'add_media_thumbnail'], "plugins" : ['kqed', 'content_encoded', 'wfw_slash_comments', 'add_media_thumbnail'],
"meta" : { "meta" : {
"title": "Test File Feed", "title": "Test File Feed",
"description": "This feed comes from a file", "description": "This feed comes from a file",

View File

@ -6,7 +6,7 @@ var test = require('tape'),
// lastBuildDate will always be this value // lastBuildDate will always be this value
var mockdate = require('mockdate').set('Wed, 31 Dec 2014 00:00:01 GMT'); var mockdate = require('mockdate').set('Wed, 31 Dec 2014 00:00:01 GMT');
test('generate feed. No plugins', function(t) { test('braid feed from file without plugins', function(t) {
t.plan(1); t.plan(1);
var feeds = {}; var feeds = {};
feeds.sample_feed = require("./feeds/sample_feed").feed; feeds.sample_feed = require("./feeds/sample_feed").feed;
@ -23,10 +23,11 @@ test('generate feed. No plugins', function(t) {
} }
// console.log(data); // console.log(data);
t.equal(data, expectedOutput.fileFeedOutput); t.equal(data, expectedOutput.fileFeedOutput);
}); });
}); });
test('generate feed and process through plugins', function(t) { test('braid feed from file with plugins', function(t) {
t.plan(1); t.plan(1);
var feeds = {}; var feeds = {};
feeds.sample_feed = require("./feeds/sample_feed_plugins").feed; feeds.sample_feed = require("./feeds/sample_feed_plugins").feed;
@ -34,7 +35,7 @@ test('generate feed and process through plugins', function(t) {
feeds : feeds, feeds : feeds,
indent : " ", indent : " ",
date_sort_order : "desc", date_sort_order : "desc",
plugins_directories : [__dirname + '/../examples/plugins/'] plugins_directories : [__dirname + '/../lib/example_plugins/']
}; };
var rss_braider = RssBraider.createClient(braider_options); var rss_braider = RssBraider.createClient(braider_options);
@ -47,7 +48,7 @@ test('generate feed and process through plugins', function(t) {
}); });
}); });
test('de-duplicate feed', function(t) { test('deduplicate feed from file', function(t) {
t.plan(1); t.plan(1);
var feeds = {}; var feeds = {};
feeds.sample_feed = require("./feeds/sample_feed_duplicates").feed; feeds.sample_feed = require("./feeds/sample_feed_duplicates").feed;
@ -55,10 +56,9 @@ test('de-duplicate feed', function(t) {
feeds : feeds, feeds : feeds,
indent : " ", indent : " ",
dedupe_fields : ["title", "guid"], dedupe_fields : ["title", "guid"],
plugins_directories : [__dirname + '/../examples/plugins/'] plugins_directories : [__dirname + '/../lib/example_plugins/']
}; };
var rss_braider = RssBraider.createClient(braider_options); var rss_braider = RssBraider.createClient(braider_options);
rss_braider.logger.level('info');
rss_braider.processFeed('sample_feed', 'rss', function(err, data){ rss_braider.processFeed('sample_feed', 'rss', function(err, data){
if (err) { if (err) {
@ -69,7 +69,7 @@ test('de-duplicate feed', function(t) {
}); });
}); });
test('sort feed articles by date descending', function(t) { test('sort by date desc', function(t) {
t.plan(1); t.plan(1);
var feeds = {}; var feeds = {};
feeds.sample_feed = require("./feeds/date_sort").feed; feeds.sample_feed = require("./feeds/date_sort").feed;
@ -77,7 +77,7 @@ test('sort feed articles by date descending', function(t) {
feeds : feeds, feeds : feeds,
indent : " ", indent : " ",
date_sort_order : "desc", date_sort_order : "desc",
plugins_directories : [__dirname + '/../examples/plugins/'] plugins_directories : [__dirname + '/../lib/example_plugins/']
}; };
var rss_braider = RssBraider.createClient(braider_options); var rss_braider = RssBraider.createClient(braider_options);
@ -90,7 +90,7 @@ test('sort feed articles by date descending', function(t) {
}); });
}); });
test('sort feed articles by date ascending', function(t) { test('sort by date asc', function(t) {
t.plan(1); t.plan(1);
var feeds = {}; var feeds = {};
feeds.sample_feed = require("./feeds/date_sort").feed; feeds.sample_feed = require("./feeds/date_sort").feed;
@ -98,7 +98,7 @@ test('sort feed articles by date ascending', function(t) {
feeds : feeds, feeds : feeds,
indent : " ", indent : " ",
date_sort_order : "asc", date_sort_order : "asc",
plugins_directories : [__dirname + '/../examples/plugins/'] plugins_directories : [__dirname + '/../lib/example_plugins/']
}; };
var rss_braider = RssBraider.createClient(braider_options); var rss_braider = RssBraider.createClient(braider_options);
@ -111,46 +111,4 @@ test('sort feed articles by date ascending', function(t) {
}); });
}); });
test('filter all articles out using plugin', function(t) {
t.plan(1);
var feeds = {};
feeds.sample_feed = require("./feeds/no_elements").feed;
var braider_options = {
feeds : feeds,
indent : " ",
date_sort_order : "asc",
plugins_directories : [__dirname + '/../examples/plugins/']
};
var rss_braider = RssBraider.createClient(braider_options);
rss_braider.logger.level('info');
rss_braider.processFeed('sample_feed', 'rss', function(err, data){
if (err) {
return t.fail(err);
}
// console.log(data);
t.equal(data, expectedOutput.emptyFeed);
});
});
test("Don't break when a filter fails and returns null", function(t) {
t.plan(1);
var feeds = {};
feeds.sample_feed = require("./feeds/sample_feed_bad_plugin").feed;
var braider_options = {
feeds : feeds,
indent : " ",
date_sort_order : "asc",
plugins_directories : [__dirname + '/../examples/plugins/']
};
var rss_braider = RssBraider.createClient(braider_options);
rss_braider.logger.level('info');
rss_braider.processFeed('sample_feed', 'rss', function(err, data){
if (err) {
return t.fail(err);
}
// console.log(data);
t.equal(data, expectedOutput.fileFeedBadPlugin);
});
});