martin/Rinser
martin/Rinser
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

reverting

Browse Source
This commit is contained in:
Martin Donnelly 2015-07-31 12:47:21 +01:00
parent 71754f138e
commit 0a79c9611a
4 changed files with 213 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.idea/sonarIssues.xml
View File

@ -3,6 +3,11 @@
<component name="issues">
<option name="index">
<map>
<entry key="/Dummy.txt">
<value>
<set />
</value>
</entry>
<entry key="/a.dummy">
<value>
<set />
@ -18,6 +23,11 @@
<set />
</value>
</entry>
<entry key="$PROJECT_DIR$/html/lifestyle.json">
<value>
<set />
</value>
</entry>
<entry key="$PROJECT_DIR$/html/lifestyle.xml">
<value>
<set />

249
.idea/workspace.xml
View File

@ -3,10 +3,7 @@
<component name="ChangeListManager">
<list default="true" id="c2fd5632-2143-4c7e-8b0d-96d7692b700b" name="Default" comment="">
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/node_modules/rss-braider/lib/RssBraider.js" afterPath="$PROJECT_DIR$/node_modules/rss-braider/lib/RssBraider.js" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/html/paleo.html" afterPath="$PROJECT_DIR$/html/paleo.html" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/paleo.js" afterPath="$PROJECT_DIR$/paleo.js" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/html/paleo.json" afterPath="$PROJECT_DIR$/html/paleo.json" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/html/paleo.xml" afterPath="$PROJECT_DIR$/html/paleo.xml" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/lifestyle.js" afterPath="$PROJECT_DIR$/lifestyle.js" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/.idea/sonarIssues.xml" afterPath="$PROJECT_DIR$/.idea/sonarIssues.xml" />
<change type="MODIFICATION" beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" />
</list>
@ -43,7 +40,7 @@
<entry file="file://$PROJECT_DIR$/paleo.js">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.0">
<caret line="85" column="29" selection-start-line="85" selection-start-column="29" selection-end-line="85" selection-end-column="29" />
<caret line="97" column="0" selection-start-line="97" selection-start-column="0" selection-end-line="97" selection-end-column="0" />
<folding />
</state>
</provider>
@ -51,9 +48,39 @@
</file>
<file leaf-file-name="lifestyle.js" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/lifestyle.js">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="-4.4827585">
<caret line="53" column="11" selection-start-line="48" selection-start-column="9" selection-end-line="53" selection-end-column="11" />
<folding />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="lifestyle.xml" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/html/lifestyle.xml">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.0">
<caret line="213" column="29" selection-start-line="213" selection-start-column="29" selection-end-line="213" selection-end-column="29" />
<caret line="888" column="92" selection-start-line="888" selection-start-column="92" selection-end-line="888" selection-end-column="92" />
<folding />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="lifestyle.html" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/html/lifestyle.html">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="28.0">
<caret line="0" column="0" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="lifestyle.json" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/html/lifestyle.json">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="-0.0">
<caret line="0" column="0" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
@ -63,17 +90,17 @@
<entry file="file://$PROJECT_DIR$/output.txt">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.0">
<caret line="15" column="13" selection-start-line="15" selection-start-column="0" selection-end-line="15" selection-end-column="13" />
<caret line="0" column="13" selection-start-line="0" selection-start-column="13" selection-end-line="0" selection-end-column="13" />
<folding />
</state>
</provider>
</entry>
</file>
<file leaf-file-name="RssBraider.js" pinned="false" current-in-tab="false">
<file leaf-file-name="RssBraider.js" pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/node_modules/rss-braider/lib/RssBraider.js">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="-4.206897">
<caret line="151" column="16" selection-start-line="151" selection-start-column="16" selection-end-line="151" selection-end-column="16" />
<state vertical-scroll-proportion="-3.2142856">
<caret line="135" column="9" selection-start-line="135" selection-start-column="9" selection-end-line="135" selection-end-column="9" />
<folding />
</state>
</provider>
@ -89,10 +116,10 @@
</provider>
</entry>
</file>
<file leaf-file-name="paleo.json" pinned="false" current-in-tab="true">
<file leaf-file-name="paleo.json" pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/html/paleo.json">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.0">
<state vertical-scroll-proportion="-0.0">
<caret line="0" column="0" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
@ -128,10 +155,11 @@
<option value="$PROJECT_DIR$/jobs-local.xml" />
<option value="$PROJECT_DIR$/cron.js" />
<option value="$PROJECT_DIR$/jobs-local.js" />
<option value="$PROJECT_DIR$/lifestyle.js" />
<option value="$PROJECT_DIR$/paleo.js" />
<option value="$PROJECT_DIR$/node_modules/rss-braider/lib/RssBraider.js" />
<option value="$PROJECT_DIR$/html/paleo.json" />
<option value="$PROJECT_DIR$/html/lifestyle.json" />
<option value="$PROJECT_DIR$/node_modules/rss-braider/lib/RssBraider.js" />
<option value="$PROJECT_DIR$/lifestyle.js" />
</list>
</option>
</component>
@ -293,6 +321,22 @@
<property name="settings.editor.splitter.proportion" value="0.2" />
<property name="last_opened_file_path" value="$PROJECT_DIR$" />
<property name="restartRequiresConfirmation" value="true" />
<property name="FileHistory.git4idea.history.GitHistoryProvider_flatWidth0" value="157" />
<property name="FileHistory.git4idea.history.GitHistoryProvider_flatOrder0" value="0" />
<property name="FileHistory.git4idea.history.GitHistoryProvider_flatWidth1" value="186" />
<property name="FileHistory.git4idea.history.GitHistoryProvider_flatOrder1" value="1" />
<property name="FileHistory.git4idea.history.GitHistoryProvider_flatWidth2" value="189" />
<property name="FileHistory.git4idea.history.GitHistoryProvider_flatOrder2" value="2" />
<property name="FileHistory.git4idea.history.GitHistoryProvider_flatWidth3" value="862" />
<property name="FileHistory.git4idea.history.GitHistoryProvider_flatOrder3" value="3" />
<property name="FileHistory.git4idea.history.GitHistoryProvider_treeWidth0" value="180" />
<property name="FileHistory.git4idea.history.GitHistoryProvider_treeOrder0" value="0" />
<property name="FileHistory.git4idea.history.GitHistoryProvider_treeWidth1" value="179" />
<property name="FileHistory.git4idea.history.GitHistoryProvider_treeOrder1" value="1" />
<property name="FileHistory.git4idea.history.GitHistoryProvider_treeWidth2" value="180" />
<property name="FileHistory.git4idea.history.GitHistoryProvider_treeOrder2" value="2" />
<property name="FileHistory.git4idea.history.GitHistoryProvider_treeWidth3" value="855" />
<property name="FileHistory.git4idea.history.GitHistoryProvider_treeOrder3" value="3" />
</component>
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
@ -313,7 +357,8 @@
<option name="mySupportsDeletion" value="false" />
<option name="myCanBeAbsent" value="false" />
<option name="myId" value="UPDATED" />
<PATH vcs="Git" revision="">$PROJECT_DIR$/output.txt</PATH>
<PATH vcs="Git" revision="">$PROJECT_DIR$/html/lifestyle.json</PATH>
<PATH vcs="Git" revision="">$PROJECT_DIR$/html/lifestyle.xml</PATH>
</FILE-GROUP>
<FILE-GROUP>
<option name="myUpdateName" value="Created" />
@ -321,6 +366,7 @@
<option name="mySupportsDeletion" value="false" />
<option name="myCanBeAbsent" value="false" />
<option name="myId" value="CREATED" />
<PATH vcs="Git" revision="">$PROJECT_DIR$/html/lifestyle.html</PATH>
</FILE-GROUP>
<FILE-GROUP>
<option name="myUpdateName" value="Deleted" />
@ -484,7 +530,67 @@
<option name="project" value="LOCAL" />
<updated>1438336477190</updated>
</task>
<option name="localTasksCounter" value="3" />
<task id="LOCAL-00003" summary="debugging braider">
<created>1438339548681</created>
<option name="number" value="00003" />
<option name="project" value="LOCAL" />
<updated>1438339548681</updated>
</task>
<task id="LOCAL-00004" summary="debugging braider">
<created>1438339596747</created>
<option name="number" value="00004" />
<option name="project" value="LOCAL" />
<updated>1438339596747</updated>
</task>
<task id="LOCAL-00005" summary="debugging braider">
<created>1438339764867</created>
<option name="number" value="00005" />
<option name="project" value="LOCAL" />
<updated>1438339764868</updated>
</task>
<task id="LOCAL-00006" summary="debugging braider">
<created>1438340241019</created>
<option name="number" value="00006" />
<option name="project" value="LOCAL" />
<updated>1438340241019</updated>
</task>
<task id="LOCAL-00007" summary="debugging braider">
<created>1438340434781</created>
<option name="number" value="00007" />
<option name="project" value="LOCAL" />
<updated>1438340434781</updated>
</task>
<task id="LOCAL-00008" summary="debugging braider">
<created>1438342430087</created>
<option name="number" value="00008" />
<option name="project" value="LOCAL" />
<updated>1438342430087</updated>
</task>
<task id="LOCAL-00009" summary="debugging braider">
<created>1438342559722</created>
<option name="number" value="00009" />
<option name="project" value="LOCAL" />
<updated>1438342559722</updated>
</task>
<task id="LOCAL-00010" summary="debugging braider">
<created>1438342805493</created>
<option name="number" value="00010" />
<option name="project" value="LOCAL" />
<updated>1438342805493</updated>
</task>
<task id="LOCAL-00011" summary="debugging braider">
<created>1438343027639</created>
<option name="number" value="00011" />
<option name="project" value="LOCAL" />
<updated>1438343027639</updated>
</task>
<task id="LOCAL-00012" summary="debugging braider">
<created>1438343085237</created>
<option name="number" value="00012" />
<option name="project" value="LOCAL" />
<updated>1438343085237</updated>
</task>
<option name="localTasksCounter" value="13" />
<servers />
</component>
<component name="ToolWindowManager">
@ -497,8 +603,8 @@
<window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
<window_info id="Application Servers" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="SonarQube" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.32962137" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Version Control" active="true" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" weight="0.32962137" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Version Control" active="true" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" weight="0.5623608" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.32962137" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
<window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
<window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
<window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" />
@ -512,6 +618,14 @@
<window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
</layout>
</component>
<component name="Vcs.Log.UiProperties">
<option name="RECENTLY_FILTERED_USER_GROUPS">
<collection />
</option>
<option name="RECENTLY_FILTERED_BRANCH_GROUPS">
<collection />
</option>
</component>
<component name="VcsContentAnnotationSettings">
<option name="myLimit" value="2678400000" />
</component>
@ -519,7 +633,8 @@
<option name="CHECK_CODE_SMELLS_BEFORE_PROJECT_COMMIT" value="false" />
<option name="CHECK_NEW_TODO" value="false" />
<MESSAGE value="debugging braider" />
<option name="LAST_COMMIT_MESSAGE" value="debugging braider" />
<MESSAGE value="reverting" />
<option name="LAST_COMMIT_MESSAGE" value="reverting" />
</component>
<component name="XDebuggerManager">
<breakpoint-manager>
@ -813,14 +928,6 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/html/lifestyle.xml">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="27.250872">
<caret line="888" column="107" selection-start-line="888" selection-start-column="107" selection-end-line="888" selection-end-column="107" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/paleo.xml">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.0">
@ -829,22 +936,6 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/lifestyle.js">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.0">
<caret line="213" column="29" selection-start-line="213" selection-start-column="29" selection-end-line="213" selection-end-column="29" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/output.txt">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.0">
<caret line="15" column="13" selection-start-line="15" selection-start-column="0" selection-end-line="15" selection-end-column="13" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$USER_HOME$/.WebStorm10/system/extLibs/nodejs-v0.12.3-src/core-modules-sources/lib/events.js">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.0">
@ -853,22 +944,6 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/paleo.js">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.0">
<caret line="85" column="29" selection-start-line="85" selection-start-column="29" selection-end-line="85" selection-end-column="29" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/node_modules/rss-braider/lib/RssBraider.js">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="-4.206897">
<caret line="151" column="16" selection-start-line="151" selection-start-column="16" selection-end-line="151" selection-end-column="16" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/html/paleo.json">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.0">
@ -877,5 +952,61 @@
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/output.txt">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.0">
<caret line="0" column="13" selection-start-line="0" selection-start-column="13" selection-end-line="0" selection-end-column="13" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/paleo.js">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.0">
<caret line="97" column="0" selection-start-line="97" selection-start-column="0" selection-end-line="97" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/html/lifestyle.json">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.0">
<caret line="0" column="0" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/html/lifestyle.xml">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="0.0">
<caret line="888" column="92" selection-start-line="888" selection-start-column="92" selection-end-line="888" selection-end-column="92" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/html/lifestyle.html">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="28.0">
<caret line="0" column="0" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/lifestyle.js">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="-4.4827585">
<caret line="53" column="11" selection-start-line="48" selection-start-column="9" selection-end-line="53" selection-end-column="11" />
<folding />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/node_modules/rss-braider/lib/RssBraider.js">
<provider selected="true" editor-type-id="text-editor">
<state vertical-scroll-proportion="-3.2142856">
<caret line="135" column="9" selection-start-line="135" selection-start-column="9" selection-end-line="135" selection-end-column="9" />
<folding />
</state>
</provider>
</entry>
</component>
</project>

29
lifestyle.js
View File

@ -139,7 +139,6 @@ feeds.simple_test_feed = {
"count" : 100,
"feed_url" : "http://www.hodinkee.com/blog/rss.xml"
},
{
"count" : 100,
@ -192,7 +191,7 @@ var braider_options = {
indent: " ",
date_sort_order: "desc", // Newest first
log_level: "warn",
dedupe_fields: [ 'guid'],
dedupe_fields: ['link', 'guid'],
plugins_directories : [__dirname + "/plugins/"]
};
var rss_braider = RssBraider.createClient(braider_options);
@ -201,26 +200,13 @@ var rss_braider = RssBraider.createClient(braider_options);
rss_braider.logger.level('off');
rss_braider.processFeed('simple_test_feed', 'json', function (err, data) {
console.log('A');
if (err) {
return console.log(err);
}
console.log('B');
/* var j;
try{
j = JSON.parse(data);
}catch(e){
console.log('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!');
console.log(e);
console.log('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!');
}*/
var j = JSON.parse(data);
var ejsOutput = ejs.compile(str)(j);
console.log(data);
//var ejsOutput = ejs.compile(str)(j);
console.log('D');
fs.writeFile(__dirname + "/html/" + outputFile + ".json", data, function (err) {
if (err) {
return console.log(err);
@ -229,18 +215,17 @@ rss_braider.processFeed('simple_test_feed', 'json', function (err, data) {
console.log("The file was saved!");
});
/*fs.writeFile(__dirname + "/html/" + outputFile + ".html", ejsOutput, function (err) {
fs.writeFile(__dirname + "/html/" + outputFile + ".html", ejsOutput, function (err) {
if (err) {
return console.log(err);
}
console.log("The file was saved!");
});*/
});
});
/*rss_braider.processFeed('simple_test_feed', 'rss', function (err, data) {
rss_braider.processFeed('simple_test_feed', 'rss', function (err, data) {
if (err) {
return console.log(err);
}
@ -251,5 +236,5 @@ rss_braider.processFeed('simple_test_feed', 'json', function (err, data) {
console.log("The file was saved!");
});
});*/
});

97
node_modules/rss-braider/lib/RssBraider.js generated vendored
View File

@ -10,9 +10,6 @@ var FeedParser = require('feedparser'),
logger;
var RssBraider = function (options) {
this.uniques = {};
this.mdUniques = [];
this.feedCount = [];
if (!options) {
options = {};
}
@ -80,15 +77,11 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
// Process each feed source through Feedparser to get articles.
// Then process each item/article through rss-braider and any plugins
async.each(feed.sources, function(source, callback) {
console.log('+ async.each');
self.feedCount.push(source.feed_url);
console.log('## FEED COUNT: ' + self.feedCount.length);
var count = source.count || feed.default_count || 10, // Number of articles per source
url = source.feed_url || null,
file_path = source.file_path || null,
source_articles = [];
console.log('Pre FeedParser');
var feedparser = new FeedParser();
if (url) {
var req = request(url);
@ -102,12 +95,10 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
if (res.statusCode !== 200) {
return this.emit('error', 'Bad status code: ' + res.statusCode);
}
console.log('** DO FEED Parser A');
stream.pipe(feedparser);
});
} else if (file_path) {
// open file
console.log('** DO FEED Parser B');
var filestream = fs.createReadStream(file_path);
filestream.pipe(feedparser);
} else {
@ -121,7 +112,6 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
// Collect the articles from this source
feedparser.on('readable', function() {
console.log('+ readable: ' + source.feed_url);
var stream = this,
item;
@ -137,32 +127,16 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
source_articles.push(article);
}
}
console.log('- readable');
});
feedparser.on("end", function(){
console.log('+ end');
// de-dupe , date sort, and trim this feed's articles and push them into array
// console.log('** Go for dedupe');
console.log(source.feed_url);
source_articles = self.dedupe(source_articles, self.dedupe_fields);
// source_articles = self.mdDedupe(source_articles, self.dedupe_fields);
source_articles = self.date_sort(source_articles);
source_articles = source_articles.slice(0, count);
feed_articles = feed_articles.concat(source_articles);
var p = self.feedCount.indexOf(source.feed_url);
if (p > -1) {
self.feedCount.splice(p, 1);
}
console.log('~~ FEED COUNT: ' + self.feedCount.length);
console.log(self.feedCount);
callback();
console.log('- end');
});
console.log('- async.each');
},
function(err){
if (err) {
@ -170,12 +144,7 @@ RssBraider.prototype.processFeed = function(feed_name, format, callback)
return callback(err);
} else {
// Final Dedupe step and resort
console.log('** ERR Go for dedupe');
feed_articles = self.dedupe(feed_articles, self.dedupe_fields);
// feed_articles = self.mdDedupe(feed_articles, self.dedupe_fields);
feed_articles = self.date_sort(feed_articles);
// Create new feed with these articles. Follows node-rss spec
@ -281,80 +250,27 @@ RssBraider.prototype.runPlugins = function (item, itemOptions, source, feed_name
// Accepts an array of fields to dedupe on, or does a basic uniq
// operation on the articles array
// TODO, make this a plugin?
RssBraider.prototype.mdDedupe = function(articles_arr, fields)
{
var deduped_articles=[];
var self = this;
// this.mdUniques = [];
console.log('+ mdDedupe');
if ( !fields || fields.length < 1 ) {
console.log('- deDupe -- empty');
return _.uniq(articles_arr);
} else {
articles_arr.forEach(function(article){
var count = 0;
fields.forEach(function(field){
/* console.log('---');
console.log(field);
console.log(article[field]);
*/
if (self.mdUniques.indexOf(article[field]) !== -1) {
// console.log('Increase count');
count++;
} else {
// console.log('Push item');
self.mdUniques.push(article[field]);
}
});
if (count == 0) {
// it's unique
// console.log('Adding:' );
deduped_articles.push(article);
} else {
// The article matched all of another article's "dedupe" fields
// so filter it out (i.e. do nothing)
// console.log("skipping duplicate", '"' + article.title + '"', article.guid);
}
});
}
console.log('- MDdeDupe');
return deduped_articles;
};
RssBraider.prototype.dedupe = function(articles_arr, fields){
console.log('+ deDupe');
var self = this;
if ( !fields || fields.length < 1 ) {
console.log('- deDupe -- empty');
return _.uniq(articles_arr);
} else {
var deduped_articles = [];
var uniques = {},
deduped_articles = [];
articles_arr.forEach(function(article){
var count = 0;
fields.forEach(function(field){
console.log('---');
console.log(field);
console.log(article[field]);
if (!self.uniques[field]) {
console.log('Doesnt exist, empty it');
self.uniques[field] = [];
if (!uniques[field]) {
uniques[field] = [];
}
if (self.uniques[field].indexOf(article[field]) !== -1) {
console.log('Increase count');
if (uniques[field].indexOf(article[field]) !== -1) {
count++;
} else {
console.log('Push item');
self.uniques[field].push(article[field]);
uniques[field].push(article[field]);
}
});
if (count !== fields.length) {
// it's unique
console.log('Adding:' );
deduped_articles.push(article);
} else {
// The article matched all of another article's "dedupe" fields
@ -362,7 +278,6 @@ RssBraider.prototype.dedupe = function(articles_arr, fields){
self.logger.debug("skipping duplicate", '"' + article.title + '"', article.guid);
}
});
console.log('- deDupe');
return deduped_articles;
}
};