RSS-27: Adding the remaining node-rss elements to the braider feed constructor. Adding support for site_url, feed_url, fullname, shortname. Updating kqed.js plugin

This commit is contained in:
Kip Gebhardt 2015-03-02 16:22:45 -08:00
parent f3d501924e
commit e835a5a20b
11 changed files with 82 additions and 20 deletions

View File

@ -69,7 +69,7 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
async.each(feed.sources, function(source, callback) {
var count = source.count || feed.default_count || 10,
url = source.url || null,
url = source.feed_url || null,
file_path = source.file_path || null,
source_articles = [];
@ -110,10 +110,10 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
item;
while ( item = stream.read() ) {
if (source.url) {
item.source_url = source.url;
if (source.feed_url) {
item.source_url = source.feed_url;
}
var article = self.processItem(item);
var article = self.processItem(item, source);
if (article) {
source_articles.push(article);
}
@ -141,10 +141,14 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
// Create new feed with these articles
var options = {
title : feed.meta.title,
site_url : "http://www.kqed.org",
description : feed.meta.description,
generator : feed.meta.generator || 'rss-braider',
feed_url : feed.meta.url,
site_url : feed.meta.site_url || null,
feed_url : feed.meta.feed_url || null,
image_url : feed.meta.image_url || null,
webMaster : feed.meta.webMaster || null,
copyright : feed.meta.copyright || null,
categories : feed.meta.categories || null,
custom_namespaces : feed.custom_namespaces || [],
no_cdata_fields : feed.no_cdata_fields
};
@ -172,7 +176,7 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
};
// Accepts a feed-parser item and builds a node-rss itemOptions object
RssBraider.prototype.processItem = function (item) {
RssBraider.prototype.processItem = function (item, source) {
var self = this;
if (!item) {
@ -195,20 +199,22 @@ RssBraider.prototype.processItem = function (item) {
// Run the plugins specified by the "plugins" section of the
// feed config file to build out any custom elements or
// do transforms
self.runPlugins(item, itemOptions);
self.runPlugins(item, itemOptions, source);
// logger.info("returning for item.guid:" + item.guid);
return itemOptions;
};
RssBraider.prototype.runPlugins = function (item, itemOptions) {
RssBraider.prototype.runPlugins = function (item, itemOptions, source) {
var self = this,
feed_plugins = self.feed.plugins || [];
feed = self.feed,
feed_plugins = feed.plugins || [];
// Process the item through the desired feed plugins
feed_plugins.forEach(function(plugin_name){
if (self.plugins[plugin_name]) {
// logger.info("DEBUG runPlugins running " + plugin_name);
self.plugins[plugin_name](item, itemOptions);
// logger.info("DEBUG runPlugins running " + plugin_name + " for item " + item.guid + " in feed: " + feed.meta.title);
self.plugins[plugin_name](item, itemOptions, source);
} else {
logger.error("A plugin named '" + plugin_name + "' hasn't been registered");
}

View File

@ -17,4 +17,5 @@ module.exports = function (item, itemOptions) {
}
);
}
return;
};

View File

@ -1,11 +1,39 @@
// define kqed source
module.exports = function (item, itemOptions) {
if (!item || !itemOptions) {
module.exports = function (item, itemOptions, source) {
if (!item || !itemOptions || !source) {
return;
}
if (item.source_url) {
// Look for kqed namespace elements in source and add as custom elements for item
// Ex:
// <kqed:fullname>The California Report</kqed:fullname>
// <kqed:shortname>the-california-report</kqed:shortname>
// <kqed:site_url>
// http://ww2.kqed.org/news/programs/the-california-report
// </kqed:site_url>
var element;
if (source.site_url){
element = {
'kqed:site_url': source.site_url
};
itemOptions.custom_elements.push(element);
}
if (source.fullname){
element = {
'kqed:fullname': source.fullname
};
itemOptions.custom_elements.push(element);
}
// add shortname using the source 'name'
element = {
'kqed:shortname': source.name
};
itemOptions.custom_elements.push(element);
// Add the source's feed url to the kqed namespace of the item/article
if (item.feed_url) {
itemOptions.custom_elements.push(
{ 'kqed:source': item.source_url }
{ 'kqed:feed_url': item.feed_url }
);
}
};

View File

@ -3,7 +3,7 @@
<channel>
<title><![CDATA[Test File Feed]]></title>
<description>This feed comes from a file</description>
<link>http://www.kqed.org</link>
<link>http://github.com/dylang/node-rss</link>
<generator>rss-braider</generator>
<lastBuildDate>Wed, 31 Dec 2014 00:00:01 GMT</lastBuildDate>
<item>
@ -14,6 +14,8 @@
<guid isPermaLink="false">http://ww2.kqed.org/arts/2014/12/20/rent-hike-for-dance-mission-theater-has-artists-worried-about-uncertain-future/</guid>
<dc:creator><![CDATA[KQED Arts]]></dc:creator>
<pubDate>Sat, 20 Dec 2014 09:00:22 GMT</pubDate>
<kqed:fullname>A Sample Feed</kqed:fullname>
<kqed:shortname>sample_feed</kqed:shortname>
<content:encoded><![CDATA[<p>Stepping out of BART at 24th and Mission at most hours of the day, one is likely to hear the pulse of African drums, hip-hop or salsa emanating from the second-floor studios of Dance Brigade&#8217;s Dance Mission Theater. But that music may not continue forever.</p>
<p>The performance space and dance school <a href="http://ww2.kqed.org/news/2014/12/20/dance-mission-theater-rent-increase-worries-artists/" target="_self" id="rssmi_more"> &#8230;read more</a>
<p>Source:: <a href="http://ww2.kqed.org/news/2014/12/20/dance-mission-theater-rent-increase-worries-artists/" target="_self" title="Rent Hike For Dance Mission Theater Has Artists Worried About Uncertain Future">Arts News</a></p>
@ -30,6 +32,8 @@
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10220517</guid>
<dc:creator><![CDATA[Sarah Hotchkiss]]></dc:creator>
<pubDate>Sat, 20 Dec 2014 14:00:47 GMT</pubDate>
<kqed:fullname>A Sample Feed</kqed:fullname>
<kqed:shortname>sample_feed</kqed:shortname>
<content:encoded><![CDATA[<p>Until February 5, 2015, you can visit the main branch of the San Francisco Public Library and be momentarily transported through space and time to the early days of the Exploratorium via the life and work of Bob Miller, who died in 2007.</p>
<p>The transportation isn&#8217;t entirely complete, especially when unavoidable elements of the library&#8217;s physical space interrupt the installation of vitrines, wall text and objects. But reading first-hand accounts by Miller&#8217;s colleagues of his various contributions to both the Exploratorium and scientific discovery, I wished I&#8217;d wandered the museum with him at any point in his twenty year tenure. Documentation and ephemera will never be as entertaining as the man himself, but <b>Light Walk: Bob Miller and the Exploratorium</b> presents one man&#8217;s creativity, playfulness and lasting impact on an institution, leaving the viewer itching for the simple tools of his trade: sunlight, paper and an irrepressible curiosity.</p>
<p><b>Light Walk </b>is installed on the librarys fourth floor, rubbing elbows with the art, music and recreation collections. Four large cases present short anecdotes about Miller and related materials from the Exploratorium archives: photographs, magazine clippings, patent documents and museum publications. The bits and pieces coalesce into a picture of Miller &#8212; much like his <i>Light Walk</i>, an exploration of sunlight, shadow, refraction and perception builds on basic demonstrations to build a larger understanding of the natural world.</p>
@ -56,6 +60,8 @@
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10231062</guid>
<dc:creator><![CDATA[KQED Arts]]></dc:creator>
<pubDate>Sun, 21 Dec 2014 14:00:08 GMT</pubDate>
<kqed:fullname>A Sample Feed</kqed:fullname>
<kqed:shortname>sample_feed</kqed:shortname>
<content:encoded><![CDATA[<p>Tonight is midwinter, Yule, the solstice the longest night. During winter&#8217;s dark depths, San Franciscans with an urge to seek the light will find a wealth of illuminated art pieces blinking and blazing around town. Some, like <a href="http://www.metaphorm.org/portfolio/carusos-dream/" target="_blank">Carusos Dream</a>, are permanent installations. Others, like <a href="https://www.kickstarter.com/projects/foldhaus/blumen-lumen-interactive-sculpture-at-burning-man" target="_blank">Blumen Lumen</a> and a selection of illuminated works at the Exploratorium through Jan. 4, will be gone in a flash. Many, such as the <a href="http://flaminglotus.com/art/soma/" target="_blank">Flaming Lotus Girls&#8217; SOMA</a>, on Pier 14, come out of the Burning Man movement. All feel appropriate to this season, when we celebrate the sun&#8217;s return with fire.</p>
<p>If you&#8217;re interested in visiting the city&#8217;s illuminated installations, the SF Travel Association <a href="http://www.illuminatesf.com/#/content/art-map" target="_blank">has compiled a map</a> and a list of <a href="http://www.illuminatesf.com/#/content/events" target="_blank">tours and other events</a>. (KQED critic Christian Frock rates the 16 works listed on the map as ranging “from meh to spectacular&#8221; &#8212; <a href="http://ww2.kqed.org/arts/2014/12/18/bigger-brighter-better-will-san-francisco-become-a-light-art-destination/" target="_blank">read her commentary</a>. ) A <a href="http://www.discoverystreettours.com/site/?page_id=747" target="_blank">free light art bike tour this Saturday</a> will give cyclists a chance to shine. Meanwhile, enjoy Peter Ruocco&#8217;s video exploration of these dazzling works.
<div id='ctx-module' class='ctx-module-container ctx-clearfix'></div>]]></content:encoded>
@ -71,6 +77,8 @@
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10230592</guid>
<dc:creator><![CDATA[Kevin L. Jones]]></dc:creator>
<pubDate>Mon, 22 Dec 2014 14:00:32 GMT</pubDate>
<kqed:fullname>A Sample Feed</kqed:fullname>
<kqed:shortname>sample_feed</kqed:shortname>
<content:encoded><![CDATA[<p>In early November, the Yerba Buena Center for the Arts (YBCA) hosted <strong>Clas/sick Hip-Hop: 1993 Edition,</strong> a night of live music featuring over 20 MCs performing new takes on hop-hop hits from 1993, with live instrumentation arranged by two notable Bay Area composers—<a href="http://ensemblemiknawooj.com/" target="_blank">JooWan Kim</a> and <a href="http://www.ybca.org/artists/kev-choice" target="_blank">Kev Choice</a>. The year&#8217;s importance was twofold: not only did YBCA first open its doors in 1993, but hip-hop itself saw serious successes, both artistically and commercially, that led to the form becoming a part of the mainstream.</p>
<p>Songs that were performed include Snoop Dogg&#8217;s &#8220;Gin And Juice,&#8221; Digable Planets&#8217; &#8220;Rebirth of Slick (Cool Like Dat),&#8221; and many others. Read a statement from the curator of the event, Marc Bamuthi Joseph, <a href="http://goo.gl/TIgUrh" target="_blank">here</a>. (Also, we asked Joseph to pick his Top 5 hip-hop songs from 1993, which you can read <a href="http://ww2.kqed.org/arts/2014/11/05/the-top-five-hip-hop-songs-from-1993-with-marc-bamuthi-joseph/" target="_blank">here</a>.)</p>
<p>Enjoyed that? Watch this performance, by Ensemble Mik Nawooj, of the Wu-Tang Clan&#8217;s &#8220;C.R.E.A.M.&#8221; from that same night.</p>
@ -88,6 +96,8 @@
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10189953</guid>
<dc:creator><![CDATA[Michele Carlson]]></dc:creator>
<pubDate>Tue, 23 Dec 2014 14:00:41 GMT</pubDate>
<kqed:fullname>A Sample Feed</kqed:fullname>
<kqed:shortname>sample_feed</kqed:shortname>
<content:encoded><![CDATA[<p>Hands up, umbrellas out, hoodies on, fists raised &#8212; the icons of protest have long played a significant role in movements of social change and revolution. From flower power to Rosie the Riveter to the rainbow flag, icons have momentous social potency and live on as cultural currency that is exchanged, needed and protected sometimes long past the initial movements of their birth.</p>
<p>Icons are sought after and are coming to be needed more and more. Perhaps it is easier to understand an image than it is to know the world, let alone how to change it.</p>
<p>Uploading an icon to a profile picture or posting a response or article bleeds into thousands who are doing the same and suddenly an entire social media feed is parroting the same visual calls. Today, protest icons are disseminated rapidly and with such reach that, while the news is mired in arguing over factual evidence of whether or not Mike Browns hands were actually raised or not, the image has already hit the streets. There thousands of protesters begin to raise their hands and take photos with their hoodies pulled tight or make videos of themselves dumping buckets of ice over their heads.</p>

View File

@ -3,7 +3,7 @@
<channel>
<title><![CDATA[Test File Feed]]></title>
<description>This feed comes from a file</description>
<link>http://www.kqed.org</link>
<link>http://github.com/dylang/node-rss</link>
<generator>rss-braider</generator>
<lastBuildDate>Wed, 31 Dec 2014 00:00:01 GMT</lastBuildDate>
<item>
@ -13,6 +13,8 @@
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10222283</guid>
<dc:creator><![CDATA[Michael Fox]]></dc:creator>
<pubDate>Fri, 26 Dec 2014 14:00:18 GMT</pubDate>
<kqed:fullname>A Sample Feed</kqed:fullname>
<kqed:shortname>sample_feed</kqed:shortname>
<content:encoded><![CDATA[<p>The year just ending wasnt a terrible one for movies, but it will be remembered as depressingly uninspired. Earnest craftsmanship is the mantra of the moment, particularly in the risk-phobic American cinema. Consider the siege weve endured (especially since Labor Day) of serious, solid movies &#8212; including <em>Gone Girl</em>, <em>Foxcatcher</em>, <em>Exodus: Gods and Kings</em>, <em>American Sniper,</em> <em>The Imitation Game</em>, <em>Unbroken</em>, <em>A Most Violent Year</em> and <em>Still Alice &#8212; </em>that demanded our attention for long hours and repaid us with the briefest flashes of transcendent joy, insight or pathos.</p>
<p>It was no easy task summoning memorable moments from the years morass, yet here in chronological order are the sequences that, for me, best captured the vitality and intelligent power that movies are capable of expressing.</p>
<p><a href="http://ww2.kqed.org/arts/wp-content/uploads/sites/2/2014/12/ida2.jpg"><img src="http://ww2.kqed.org/arts/wp-content/uploads/sites/2/2014/12/ida2.jpg" alt="ida2" width="640" height="420" class="alignright size-full wp-image-10238620" /></a></p>
@ -59,6 +61,8 @@
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10238611</guid>
<dc:creator><![CDATA[The Do List]]></dc:creator>
<pubDate>Fri, 26 Dec 2014 02:00:10 GMT</pubDate>
<kqed:fullname>A Sample Feed</kqed:fullname>
<kqed:shortname>sample_feed</kqed:shortname>
<content:encoded><![CDATA[<aside class="aligncenter">
<h4>Cy Musiker and David Wiegand share their picks for great events around the Bay Area this week.</h4>
<p><!--[if lt IE 9]><script>document.createElement('audio');</script><![endif]-->
@ -99,6 +103,8 @@
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10238600</guid>
<dc:creator><![CDATA[Siouxsie Oki]]></dc:creator>
<pubDate>Wed, 24 Dec 2014 21:05:45 GMT</pubDate>
<kqed:fullname>A Sample Feed</kqed:fullname>
<kqed:shortname>sample_feed</kqed:shortname>
<content:encoded><![CDATA[<p>Above:<a href="http://ensemblemiknawooj.com/"> Ensemble Mik Nawooj</a> (EMN), led by music director, composer/pianist JooWan Kim, reinterprets <a href="http://www.wutang-corp.com/artists/wu-tang-clan.php">Wu-Tang Clan</a>s &#8220;C.R.E.A.M.&#8221; at <a href="http://www.ybca.org/classick-hip-hop">YBCA&#8217;s <em>Clas/Sick Hip Hop: 1993 Edition.</em></a></p>
<p><a href="http://ww2.kqed.org/arts/wp-content/uploads/sites/2/2014/12/joowan.png"><img class="alignright size-full wp-image-10238602" src="http://ww2.kqed.org/arts/wp-content/uploads/sites/2/2014/12/joowan.png" alt="joowan" width="683" height="352" /></a>Ensemble Mik Nawooj (EMN) is the brainchild of composer/pianist JooWan Kim, who introduces western-European classical techniques into hip-hop, rock, and pop. The group&#8217;s lineup includes traditional Pierrot ensemble instrumentation (flute, clarinet, violin, cello and piano) with a lyric soprano, deep funk drums, a heavy contrabass and two featured MCs. Kim is a classically trained composer who holds degrees from <a href="http://www.berklee.edu/">Berklee College of Music</a> and <a href="http://www.sfcm.edu/">San Francisco Conservatory of Music</a>.</p>
<p><em>“Nobody has ever attempted anything like this on such a large scale, or with such sophistication.”</em> NPR</p>
@ -117,6 +123,8 @@
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10238610</guid>
<dc:creator><![CDATA[Siouxsie Oki]]></dc:creator>
<pubDate>Wed, 24 Dec 2014 19:00:34 GMT</pubDate>
<kqed:fullname>A Sample Feed</kqed:fullname>
<kqed:shortname>sample_feed</kqed:shortname>
<content:encoded><![CDATA[<p>Above:<a href="http://kevchoice.wordpress.com/"> Kev Choice Ensemble</a> reinterprets Saafir&#8217;s &#8220;Light Sleeper&#8221; at <a href="http://www.ybca.org/classick-hip-hop">YBCA&#8217;s <em>Clas/Sick Hip Hop: 1993 Edition.</em></a></p>
<p>Oakland-based artist <a href="http://kevchoice.wordpress.com/">Kev Choice</a> is a pianist, MC, producer, bandleader, sideman, music historian and urban griot dedicated to cultural expression. Through his band Kev Choice Ensemble, he produces and performs his own material, including original jazz and classical compositions as well as classical, jazz, and funk-inspired hip-hop.</p>
<p><a href="http://ww2.kqed.org/arts/wp-content/uploads/sites/2/2014/12/kev.jpg"><img class="alignright size-full wp-image-10238616" src="http://ww2.kqed.org/arts/wp-content/uploads/sites/2/2014/12/kev.jpg" alt="kev" width="908" height="510" /></a></p>
@ -136,6 +144,8 @@
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10218934</guid>
<dc:creator><![CDATA[Brian Eder]]></dc:creator>
<pubDate>Wed, 24 Dec 2014 14:00:19 GMT</pubDate>
<kqed:fullname>A Sample Feed</kqed:fullname>
<kqed:shortname>sample_feed</kqed:shortname>
<content:encoded><![CDATA[<p>The intent wasnt always to create a million paintings. Speaking of his early art-making days with his brothers-in-arms back in high school, where the seeds of the idea were planted, artist Jason McHenry says, “We actually just used them like currency, kind of. We would make little mini paintings. Thats how it started&#8230; just amassing a bunch of these things, you know? We would all just push each other to make a new version of our own currency &#8212; like one dollar bills and five dollar bills.”</p>
<p>McHenry and his best friend Chris Dyer are the only two that stuck to it over the years. Now the project has a name, <em>One Thousand Thousand</em>, and a mission to produce one million paintings. Although the impetus was simply to make as much art as possible surrounded by his friends, it eventually took McHenry down an often solitary path. Today he is reaching out to other artists to participate in the project to assure the goal is reached in his lifetime.</p>
<p><em>One Thousand Thousand</em> is Christopher Dyer and Jason McHenry,” McHenry is quick to point out. “I honestly do more of the work, produce more pieces; hes a senior executive at a company in Denver. Were like partners in crime artistically. All the big projects that weve done, weve done together. So we push each other that way.”</p>

View File

@ -3,7 +3,7 @@
<channel>
<title><![CDATA[Test File Feed]]></title>
<description>This feed comes from a file</description>
<link>http://www.kqed.org</link>
<link>http://github.com/dylang/node-rss</link>
<generator>rss-braider</generator>
<lastBuildDate>Wed, 31 Dec 2014 00:00:01 GMT</lastBuildDate>
<item>

View File

@ -3,7 +3,7 @@
<channel>
<title><![CDATA[Test File Feed]]></title>
<description>This feed comes from a file</description>
<link>http://www.kqed.org</link>
<link>http://github.com/dylang/node-rss</link>
<generator>rss-braider</generator>
<lastBuildDate>Wed, 31 Dec 2014 00:00:01 GMT</lastBuildDate>
<item>
@ -13,6 +13,8 @@
<guid isPermaLink="false">http://ww2.kqed.org/arts/?p=10222283</guid>
<dc:creator><![CDATA[Michael Fox]]></dc:creator>
<pubDate>Fri, 26 Dec 2014 14:00:18 GMT</pubDate>
<kqed:fullname>A Sample Feed</kqed:fullname>
<kqed:shortname>sample_feed</kqed:shortname>
<content:encoded><![CDATA[<p>The year just ending wasnt a terrible one for movies, but it will be remembered as depressingly uninspired. Earnest craftsmanship is the mantra of the moment, particularly in the risk-phobic American cinema. Consider the siege weve endured (especially since Labor Day) of serious, solid movies &#8212; including <em>Gone Girl</em>, <em>Foxcatcher</em>, <em>Exodus: Gods and Kings</em>, <em>American Sniper,</em> <em>The Imitation Game</em>, <em>Unbroken</em>, <em>A Most Violent Year</em> and <em>Still Alice &#8212; </em>that demanded our attention for long hours and repaid us with the briefest flashes of transcendent joy, insight or pathos.</p>
<p>It was no easy task summoning memorable moments from the years morass, yet here in chronological order are the sequences that, for me, best captured the vitality and intelligent power that movies are capable of expressing.</p>
<p><a href="http://ww2.kqed.org/arts/wp-content/uploads/sites/2/2014/12/ida2.jpg"><img src="http://ww2.kqed.org/arts/wp-content/uploads/sites/2/2014/12/ida2.jpg" alt="ida2" width="640" height="420" class="alignright size-full wp-image-10238620" /></a></p>

View File

@ -20,6 +20,7 @@ var feed = {
"name" : "sample_feed",
"count" : 5,
"file_path" : __dirname + "/../input_files/sample_feed.xml",
"fullname" : "A Sample Feed"
},
]

View File

@ -20,6 +20,8 @@ var feed = {
"name" : "sample_feed",
"count" : 1,
"file_path" : __dirname + "/../input_files/sample_feed_duplicates.xml",
"fullname" : "A Sample Feed"
},
]

View File

@ -20,6 +20,7 @@ var feed = {
"name" : "sample_feed",
"count" : 1,
"file_path" : __dirname + "/../input_files/sample_feed.xml",
"fullname" : "A Sample Feed"
},
]
};

View File

@ -62,6 +62,7 @@ test('deduplicate feed from file', function(t) {
if (err) {
return t.fail(err);
}
// console.log(data);
t.equal(data, expectedOutput.fileFeedOutputPlugins);
});
});