From 1bbd658815fc9df1d9192c823c2efdd670b8045c Mon Sep 17 00:00:00 2001 From: Martin Donnelly Date: Tue, 5 Jul 2016 11:09:41 +0100 Subject: [PATCH] removed a feed from paleo and now using my own copy of rss-braider --- .idea/workspace.xml | 397 +++++++-------------- node_modules/rss-braider/.npmignore | 80 ++++- node_modules/rss-braider/lib/RssBraider.js | 349 +++++++++--------- node_modules/rss-braider/package.json | 45 +-- package.json | 2 +- paleo.js | 8 +- 6 files changed, 398 insertions(+), 483 deletions(-) diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 4c451e4..dab5781 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -1,31 +1,13 @@ - - - - - + - - - - - - - - - - - - - - + + - - @@ -48,65 +30,21 @@ - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - + + - - - - - - - - - - - - - - - - - - - - - - + + @@ -115,63 +53,38 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - + + - - - - - + + + - - + + - - + + + + + + + + + + + + @@ -220,7 +133,6 @@ @@ -319,7 +232,7 @@ - + @@ -338,23 +251,9 @@ - - - - - - - - - + @@ -363,7 +262,7 @@ - + @@ -537,28 +436,9 @@ - - - - 1438336477190 - - - 1438339548681 - - - 1438339596747 - 1438339764867 @@ -882,37 +762,58 @@ - - - - + + - - + - + + @@ -967,7 +868,9 @@ - @@ -976,13 +879,6 @@ - - - - - - - @@ -1059,14 +955,14 @@ + + + - - - @@ -1132,13 +1028,6 @@ - - - - - - - @@ -1157,7 +1046,6 @@ - @@ -1165,7 +1053,6 @@ - @@ -1173,7 +1060,6 @@ - @@ -1181,7 +1067,6 @@ - @@ -1189,7 +1074,6 @@ - @@ -1197,23 +1081,6 @@ - - - - - - - - - - - - - - - - - @@ -1221,7 +1088,6 @@ - @@ -1229,15 +1095,6 @@ - - - - - - - - - @@ -1245,7 +1102,6 @@ - @@ -1253,9 +1109,6 @@ - - - @@ -1263,17 +1116,13 @@ - - - - + - @@ -1281,7 +1130,6 @@ - @@ -1289,38 +1137,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1328,19 +1144,13 @@ - - - - + - - - @@ -1348,22 +1158,67 @@ - - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/node_modules/rss-braider/.npmignore b/node_modules/rss-braider/.npmignore index b512c09..277dd4b 100644 --- a/node_modules/rss-braider/.npmignore +++ b/node_modules/rss-braider/.npmignore @@ -1 +1,79 @@ -node_modules \ No newline at end of file +# Created by .ignore support plugin (hsz.mobi) +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio + +*.iml + +/dist +## Directory-based project format: +.idea/ +# if you remove the above rule, at least ignore the following: + +# User-specific stuff: +# .idea/workspace.xml +# .idea/tasks.xml +# .idea/dictionaries + +# Sensitive or high-churn files: +# .idea/dataSources.ids +# .idea/dataSources.xml +# .idea/sqlDataSources.xml +# .idea/dynamic.xml +# .idea/uiDesigner.xml + +# Gradle: +# .idea/gradle.xml +# .idea/libraries + +# Mongo Explorer plugin: +# .idea/mongoSettings.xml + +## File-based project format: +*.ipr +*.iws + +## Plugin-specific files: + +# IntelliJ +/out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +### Node template +# Logs +logs +*.log +npm-debug.log* + +# Runtime data +pids +*.pid +*.seed + +# Directory for instrumented libs generated by jscoverage/JSCover +lib-cov + +# Coverage directory used by tools like istanbul +coverage + +# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) +.grunt + +# node-waf configuration +.lock-wscript + +# Compiled binary addons (http://nodejs.org/api/addons.html) +build/Release + +# Dependency directory +# https://docs.npmjs.com/misc/faq#should-i-check-my-node-modules-folder-into-git +node_modules + diff --git a/node_modules/rss-braider/lib/RssBraider.js b/node_modules/rss-braider/lib/RssBraider.js index eb8c2c6..ca21660 100644 --- a/node_modules/rss-braider/lib/RssBraider.js +++ b/node_modules/rss-braider/lib/RssBraider.js @@ -1,207 +1,206 @@ -// process feed-reader item into node-rss item -var FeedParser = require('feedparser'), - bunyan = require('bunyan'), - _ = require('lodash'), - async = require('async'), - request = require('request'), - RSS = require('rss'), - fs = require('fs'), - package_json = require('../package.json'), - logger; +// Process feed-reader item into node-rss item +var FeedParser = require('feedparser'); +var bunyan = require('bunyan'); +var _ = require('lodash'); +var async = require('async'); +var request = require('request'); +var RSS = require('rss'); +var fs = require('fs'); +var package_json = require('../package.json'); +var logger; -var RssBraider = function (options) { +var RssBraider = function(options) { if (!options) { - options = {}; + options = {}; } this.feeds = options.feeds || null; this.logger = options.logger || bunyan.createLogger({name: package_json.name}); if (options.log_level) { - this.logger.level(options.log_level); + this.logger.level(options.log_level); } - this.indent = options.indent || " "; + this.indent = options.indent || ' '; this.dedupe_fields = options.dedupe_fields || []; // The fields to use to identify duplicate articles - this.date_sort_order = options.date_sort_order || "desc"; + this.date_sort_order = options.date_sort_order || 'desc'; this.plugins_directories = options.plugins_directories || []; this.plugins = {}; this.loadPlugins(); -}; + }; -// loadup self.plugins with the plugin functions -RssBraider.prototype.loadPlugins = function () { +// Loadup self.plugins with the plugin functions +RssBraider.prototype.loadPlugins = function() { var self = this; if (self.plugins_directories.length < 1) { - self.logger.debug("No plugins_directories specified. No plugins loaded."); + self.logger.debug('No plugins_directories specified. No plugins loaded.'); } - self.plugins_directories.forEach(function(path){ - // load up each file and assign it to the plugins + self.plugins_directories.forEach(function(path) { + // Load up each file and assign it to the plugins var filenames = fs.readdirSync(path); - filenames.forEach(function(filename){ + filenames.forEach(function(filename) { var plugin_name = filename.replace(/.js$/, ''); if (self.plugins[plugin_name]) { - self.logger.warn("Duplicate plugin name: ", plugin_name, "Overwriting with newer plugin"); + self.logger.warn('Duplicate plugin name: ', plugin_name, 'Overwriting with newer plugin'); } self.plugins[plugin_name] = require(path + '/' + plugin_name); - self.logger.debug("plugin registered:", plugin_name); - }); - }); -}; + self.logger.debug('plugin registered:', plugin_name); + }); + }); + }; -RssBraider.prototype.feedExists = function (feed_name) { +RssBraider.prototype.feedExists = function(feed_name) { if (this.feeds && this.feeds[feed_name]) { - return true; + return true; } else { - return false; + return false; } -}; + }; // Gather data from all feed sources, process each article/item through plugins, // trim down to desired count, dedupe and sort -RssBraider.prototype.processFeed = function(feed_name, format, callback) -{ - var self = this, - feed = self.feeds[feed_name], - feed_articles = []; +RssBraider.prototype.processFeed = function(feed_name, format, callback) { + var self = this; + var feed = self.feeds[feed_name]; + var feed_articles = []; if (!format) { - format = 'rss'; + format = 'rss'; } if (!feed || !feed.sources || feed.sources.length < 1) { - return callback("No definition for feed name: " + feed_name); + return callback('No definition for feed name: ' + feed_name); } // Process each feed source through Feedparser to get articles. // Then process each item/article through rss-braider and any plugins async.each(feed.sources, function(source, callback) { - var count = source.count || feed.default_count || 10, // Number of articles per source - url = source.feed_url || null, - file_path = source.file_path || null, - source_articles = []; + var count = source.count || feed.default_count || 10; + var url = source.feed_url || null; + var file_path = source.file_path || null; + var source_articles = []; var feedparser = new FeedParser(); if (url) { - var req = request(url); + var req = request(url); - req.on('error', function (error) { - self.logger.error(error); + req.on('error', function(error) { + self.logger.error(error); }); - req.on('response', function (res) { - var stream = this; - if (res.statusCode !== 200) { - return this.emit('error', 'Bad status code: ' + res.statusCode); - } - stream.pipe(feedparser); + req.on('response', function(res) { + var stream = this; + if (res.statusCode !== 200) { + return this.emit('error', 'Bad status code for ' + url + ': ' + res.statusCode); + } + stream.pipe(feedparser); }); } else if (file_path) { - // open file - var filestream = fs.createReadStream(file_path); - filestream.pipe(feedparser); + // Open file + var filestream = fs.createReadStream(file_path); + filestream.pipe(feedparser); } else { - self.logger.error("url or file_path not defined for feed: " + source.name); - return callback(); + self.logger.error('url or file_path not defined for feed: ' + source.name); + return callback(); } feedparser.on('error', function(error) { - self.logger.error("feedparser",", source.name:", source.name, ", url:", source.feed_url, error.stack); - }); + self.logger.error('feedparser',', source.name:', source.name, ', url:', source.feed_url, error.stack); + }); // Collect the articles from this source feedparser.on('readable', function() { var stream = this, item; - while ( !!(item = stream.read()) ) { - if (source.feed_url) { - item.source_url = source.feed_url; - } - // Process Item/Article - var article = self.processItem(item, source, feed_name); + while (!!(item = stream.read())) { + if (source.feed_url) { + item.source_url = source.feed_url; + } + // Process Item/Article + var article = self.processItem(item, source, feed_name); - // plugins may filter items and return null - if (article) { - source_articles.push(article); - } + // Plugins may filter items and return null + if (article) { + source_articles.push(article); + } } - }); + }); - feedparser.on("end", function(){ - // de-dupe , date sort, and trim this feed's articles and push them into array + feedparser.on('end', function() { + // De-dupe , date sort, and trim this feed's articles and push them into array source_articles = self.dedupe(source_articles, self.dedupe_fields); source_articles = self.date_sort(source_articles); source_articles = source_articles.slice(0, count); feed_articles = feed_articles.concat(source_articles); callback(); - }); - }, - function(err){ + }); + }, + function(err) { if (err) { - self.logger.error(err); - return callback(err); + self.logger.error(err); + return callback(err); } else { - // Final Dedupe step and resort - feed_articles = self.dedupe(feed_articles, self.dedupe_fields); - feed_articles = self.date_sort(feed_articles); + // Final Dedupe step and resort + feed_articles = self.dedupe(feed_articles, self.dedupe_fields); + feed_articles = self.date_sort(feed_articles); - // Create new feed with these articles. Follows node-rss spec - var options = { - title : feed.meta.title, - description : feed.meta.description, - generator : feed.meta.generator || 'rss-braider', - site_url : feed.meta.site_url || null, - feed_url : feed.meta.feed_url || null, - image_url : feed.meta.image_url || null, - webMaster : feed.meta.webMaster || null, - copyright : feed.meta.copyright || null, - categories : feed.meta.categories || null, - custom_namespaces : feed.custom_namespaces || [], - no_cdata_fields : feed.no_cdata_fields + // Create new feed with these articles. Follows node-rss spec + var options = { + title: feed.meta.title, + description: feed.meta.description, + generator: feed.meta.generator || 'rss-braider', + site_url: feed.meta.site_url || null, + feed_url: feed.meta.feed_url || null, + image_url: feed.meta.image_url || null, + webMaster: feed.meta.webMaster || null, + copyright: feed.meta.copyright || null, + categories: feed.meta.categories || null, + custom_namespaces: feed.custom_namespaces || [], + no_cdata_fields: feed.no_cdata_fields }; - var newfeed = new RSS(options, feed_articles); + var newfeed = new RSS(options, feed_articles); - var ret_string; - switch (format.toLowerCase()) { - case 'json': - ret_string = JSON.stringify(newfeed); - break; - case 'rss': - ret_string = newfeed.xml(self.indent); - break; - default: - self.logger.error("Unknown format:", format); - ret_string = "{}"; - } - - return callback(null, ret_string); + var ret_string; + switch (format.toLowerCase()) { + case 'json': + ret_string = JSON.stringify(newfeed); + break; + case 'rss': + ret_string = newfeed.xml(self.indent); + break; + default: + self.logger.error('Unknown format:', format); + ret_string = '{}'; } - }); -}; + + return callback(null, ret_string); + } + }); + }; // Accepts a feed-parser item and builds a node-rss itemOptions object -RssBraider.prototype.processItem = function (item, source, feed_name) { +RssBraider.prototype.processItem = function(item, source, feed_name) { var self = this; if (!item || !source || !feed_name) { - self.logger.error("processItem: missing item, source, and/or feed_name"); - return null; + self.logger.error('processItem: missing item, source, and/or feed_name'); + return null; } // Basics var itemOptions = { - title : item.title, - description : item.summary, - url : item.link, - guid : item.guid, - permalink : item.permalink, - author : item.author, - date : item.date, - categories : item.categories, - custom_elements : [] - }; + title: item.title, + description: item.summary, + url: item.link, + guid: item.guid, + permalink: item.permalink, + author: item.author, + date: item.date, + categories: item.categories, + custom_elements: [] + }; // Run the plugins specified by the "plugins" section of the // feed .js file to build out any custom elements or @@ -209,9 +208,9 @@ RssBraider.prototype.processItem = function (item, source, feed_name) { var filteredItemOptions = self.runPlugins(item, itemOptions, source, feed_name); return filteredItemOptions; -}; + }; -RssBraider.prototype.runPlugins = function (item, itemOptions, source, feed_name) { +RssBraider.prototype.runPlugins = function(item, itemOptions, source, feed_name) { var self = this, feed = self.feeds[feed_name] || {}, plugins_list = feed.plugins || [], @@ -221,77 +220,77 @@ RssBraider.prototype.runPlugins = function (item, itemOptions, source, feed_name // Process the item through the desired feed plugins // plugins_list.forEach(function(plugin_name){ for (var i = 0; i < plugins_list.length; i++) { - var plugin_name = plugins_list[i]; - if (self.plugins[plugin_name]) { - filteredItemOptions = self.plugins[plugin_name](item, itemOptions, source); - } else { - self.logger.error("A plugin named '" + plugin_name + "' hasn't been registered"); - } + var plugin_name = plugins_list[i]; + if (self.plugins[plugin_name]) { + filteredItemOptions = self.plugins[plugin_name](item, itemOptions, source); + } else { + self.logger.error('A plugin named \'' + plugin_name + '\' hasn\'t been registered'); + } - // A plugin returning -1 means skip this item - if (filteredItemOptions === -1) { - self.logger.debug("Plugin '" + plugin_name + "' filtered item from feed '" + feed.meta.title + "'", item.guid); - itemOptions = null; - break; - } + // A plugin returning -1 means skip this item + if (filteredItemOptions === -1) { + self.logger.debug('Plugin \'' + plugin_name + '\' filtered item from feed \'' + feed.meta.title + '\'', item.guid); + itemOptions = null; + break; + } - // Check that the plugin didn't just return null or undef, which would be bad. - if (!filteredItemOptions) { - self.logger.debug("Plugin '" + plugin_name + "' failed to return itemOptions for feed:'" + feed.meta.title + "'", item.guid); - filteredItemOptions = itemOptions; // Reset - } - // Prepare for next plugin. - itemOptions = filteredItemOptions; + // Check that the plugin didn't just return null or undef, which would be bad. + if (!filteredItemOptions) { + self.logger.debug('Plugin \'' + plugin_name + '\' failed to return itemOptions for feed:\'' + feed.meta.title + '\'', item.guid); + filteredItemOptions = itemOptions; // Reset + } + // Prepare for next plugin. + itemOptions = filteredItemOptions; } return itemOptions; -}; + }; // Dedupe articles in node-rss itemOptions format // Accepts an array of fields to dedupe on, or does a basic uniq // operation on the articles array // TODO, make this a plugin? -RssBraider.prototype.dedupe = function(articles_arr, fields){ +RssBraider.prototype.dedupe = function(articles_arr, fields) { var self = this; - if ( !fields || fields.length < 1 ) { - return _.uniq(articles_arr); + if (!fields || fields.length < 1) { + return _.uniq(articles_arr); } else { - var uniques = {}, - deduped_articles = []; - articles_arr.forEach(function(article){ - var count = 0; - fields.forEach(function(field){ - if (!uniques[field]) { - uniques[field] = []; - } - if (uniques[field].indexOf(article[field]) !== -1) { - count++; - } else { - uniques[field].push(article[field]); - } + var uniques = {}, + deduped_articles = []; + articles_arr.forEach(function(article) { + var count = 0; + fields.forEach(function(field) { + if (!uniques[field]) { + uniques[field] = []; + } + if (uniques[field].indexOf(article[field]) !== -1) { + count++; + } else { + uniques[field].push(article[field]); + } }); - if (count !== fields.length) { - // it's unique - deduped_articles.push(article); - } else { - // The article matched all of another article's "dedupe" fields - // so filter it out (i.e. do nothing) - self.logger.debug("skipping duplicate", '"' + article.title + '"', article.guid); - } + if (count !== fields.length) { + // It's unique + deduped_articles.push(article); + } else { + // The article matched all of another article's "dedupe" fields + // so filter it out (i.e. do nothing) + self.logger.debug('skipping duplicate', '"' + article.title + '"', article.guid); + } }); - return deduped_articles; + return deduped_articles; } -}; + }; // TODO: Could be a plugin // Sort articles by date RssBraider.prototype.date_sort = function(articles_arr) { var sorted_articles = _.sortBy(articles_arr, function(article) { return article.date.getTime(); - }); - if (this.date_sort_order === "desc") { - sorted_articles.reverse(); + }); + if (this.date_sort_order === 'desc') { + sorted_articles.reverse(); } return sorted_articles; -}; + }; -module.exports = RssBraider; \ No newline at end of file +module.exports = RssBraider; diff --git a/node_modules/rss-braider/package.json b/node_modules/rss-braider/package.json index 9b3c165..1810df7 100644 --- a/node_modules/rss-braider/package.json +++ b/node_modules/rss-braider/package.json @@ -1,37 +1,31 @@ { "_args": [ [ - "rss-braider@^1.0.0", + "git+http://gitlab.silvrtree.co.uk/martind2000/rss-braider.git", "/Users/martin/newdev/Rinser" ] ], - "_from": "rss-braider@>=1.0.0 <2.0.0", + "_from": "git+http://gitlab.silvrtree.co.uk/martind2000/rss-braider.git", "_id": "rss-braider@1.0.0", "_inCache": true, "_installable": true, "_location": "/rss-braider", - "_nodeVersion": "0.10.38", - "_npmUser": { - "email": "kgebhardt23@gmail.com", - "name": "rv-kip" - }, - "_npmVersion": "2.7.6", "_phantomChildren": {}, "_requested": { - "name": "rss-braider", - "raw": "rss-braider@^1.0.0", - "rawSpec": "^1.0.0", + "name": null, + "raw": "git+http://gitlab.silvrtree.co.uk/martind2000/rss-braider.git", + "rawSpec": "git+http://gitlab.silvrtree.co.uk/martind2000/rss-braider.git", "scope": null, - "spec": ">=1.0.0 <2.0.0", - "type": "range" + "spec": "http://gitlab.silvrtree.co.uk/martind2000/rss-braider.git", + "type": "git" }, "_requiredBy": [ "#DEV:/" ], - "_resolved": "http://registry.npmjs.org/rss-braider/-/rss-braider-1.0.0.tgz", - "_shasum": "1ea5ddf8d39151728a5a039af5b0c3407f3e464b", + "_resolved": "git+http://gitlab.silvrtree.co.uk/martind2000/rss-braider.git#1559131d2e2d73076307667d260f0e173e64c8c3", + "_shasum": "f76e869410d78f6c524443695e391a92b7c0588d", "_shrinkwrap": null, - "_spec": "rss-braider@^1.0.0", + "_spec": "git+http://gitlab.silvrtree.co.uk/martind2000/rss-braider.git", "_where": "/Users/martin/newdev/Rinser", "author": { "email": "kgebhardt@kqed.org", @@ -55,13 +49,7 @@ "mockdate": "^1.0.3", "tape": "^4.0.0" }, - "directories": {}, - "dist": { - "shasum": "1ea5ddf8d39151728a5a039af5b0c3407f3e464b", - "tarball": "https://registry.npmjs.org/rss-braider/-/rss-braider-1.0.0.tgz" - }, - "gitHead": "755970200088eab93a8a8ae15b47a11b9fbb5880", - "homepage": "https://github.com/KQED/rss-braider", + "gitHead": "1559131d2e2d73076307667d260f0e173e64c8c3", "keywords": [ "aggregator", "atom", @@ -82,18 +70,13 @@ ], "license": "MIT", "main": "index.js", - "maintainers": [ - { - "name": "rv-kip", - "email": "kgebhardt23@gmail.com" - } - ], "name": "rss-braider", "optionalDependencies": {}, - "readme": "ERROR: No README data found!", + "readme": "[![Build Status](https://travis-ci.org/KQED/rss-braider.svg?branch=master)](https://travis-ci.org/KQED/rss-braider)\n\n## Summary\nBraid/aggregate one or more RSS feeds (file or url) into a single feed (RSS or JSON output). Process resulting feed through specified plugins. Automatic deduplication\n\n## Installation\n```\nnpm install rss-braider\n```\n## Test\n`npm test`\n\n## Examples\n```\n$ cd examples\n$ node simple.js (combines 3 sources)\n$ node plugins.js (combines 3 sources and runs a transformation plugin)\n```\n### Code Example\n```js\nvar RssBraider = require('rss-braider'),\n feeds = {};\n\n// Pull feeds from config files:\n// feeds.simple_test_feed = require(\"./config/feed\").feed;\n// Or define in-line\nfeeds.simple_test_feed = {\n \"feed_name\" : \"feed\",\n \"default_count\" : 1,\n \"no_cdata_fields\" : [], // Don't wrap these fields in CDATA tags\n \"meta\" : {\n \"title\": \"NPR Braided Feed\",\n \"description\": \"This is a test of two NPR\"\n },\n \"sources\" : [\n {\n \"name\" : \"NPR Headlines\",\n \"count\" : 2,\n \"feed_url\" : \"http://www.npr.org/rss/rss.php?id=1001\",\n },\n {\n \"name\" : \"NPR Sports\",\n \"count\" : 2,\n \"feed_url\" : \"http://www.npr.org/rss/rss.php?id=1055\"\n }\n ]\n};\nvar braider_options = {\n feeds : feeds,\n indent : \" \",\n date_sort_order : \"desc\", // Newest first\n log_level : \"debug\"\n};\nvar rss_braider = RssBraider.createClient(braider_options);\n\n// Override logging level (debug, info, warn, err, off)\nrss_braider.logger.level('off');\n\n// Output braided feed as rss. use 'json' for JSON output.\nrss_braider.processFeed('simple_test_feed', 'rss', function(err, data){\n if (err) {\n return console.log(err);\n }\n console.log(data);\n});\n```\n## Plugins\nPlugins provide custom manipulation and filtering of RSS items/articles. See `examples/plugins` for examples.\n\nA plugin operates by modifying the `itemOptions` object or by returning `null` which will exclude the `item` (article) from the resulting feed (See `examples/plugins/filter_out_all_articles.js`).\n\nThe `itemsOptions` object gets passed to `node-rss` to generate the RSS feeds, so read the documentation on that module and its use of custom namespaces. (https://github.com/dylang/node-rss)\n\n### Plugin Example\nThis plugin will capitalize the article title for all articles\n```js\nmodule.exports = function (item, itemOptions, source) {\n if (!item || !itemOptions) {\n return;\n }\n\n if (itemOptions.title) {\n itemOptions.title = itemOptions.title.toUpperCase();\n }\n\n return itemOptions;\n};\n```\n\nThe plugin is registered with the feed in the feed config .js file and are run in order.\n```js\nvar feed = {\n \"feed_name\" : \"feed with plugins\",\n \"default_count\" : 1,\n \"plugins\" : ['capitalize_title', 'plugin_template'],\n...\n```\n\n## Release Notes\n### 1.0.0\nChanged plugin architecture to allow filtering out of article/items by returning `-1` instead of a modified `itemsOptions` object. This is a breaking change as it will require existing plugins to return `itemsOptions` instead of modifying the reference. See `examples/plugins`.\n\n", + "readmeFilename": "README.md", "repository": { "type": "git", - "url": "git+https://github.com/KQED/rss-braider.git" + "url": "git+http://gitlab.silvrtree.co.uk/martind2000/rss-braider.git" }, "scripts": { "test": "tape test" diff --git a/package.json b/package.json index 8d81345..6be4a04 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "jshint": "^2.9.1", "lodash": "^4.11.1", "log4js": "^0.6.35", - "rss-braider": "^1.0.0" + "rss-braider": "git+http://gitlab.silvrtree.co.uk/martind2000/rss-braider.git" }, "dependencies": { "express": "^4.13.4", diff --git a/paleo.js b/paleo.js index da55610..d3710a0 100644 --- a/paleo.js +++ b/paleo.js @@ -57,11 +57,11 @@ feeds.simple_test_feed = { "count": 100, "feed_url": "http://www.ruled.me/feed/" - }, { + }/*, { "count": 100, "feed_url": "http://www.primalbro.com/blog?format=RSS" - } + }*/ @@ -74,14 +74,14 @@ var braider_options = { feeds: feeds, indent: " ", date_sort_order: "desc", // Newest first - log_level: "warn", + log_level: "error", dedupe_fields: ['guid'], plugins_directories: [__dirname + "/plugins/"] }; var rss_braider = RssBraider.createClient(braider_options); // Override logging level (debug, info, warn, err, off) -rss_braider.logger.level('debug'); +rss_braider.logger.level('error'); rss_braider.processFeed('simple_test_feed', 'json', function(err, data) { if (err) {