added a new script filter and no idea wtf is up with node

This commit is contained in:
Martin Donnelly 2015-08-14 10:55:39 +01:00
parent d73a93ec9e
commit ab44d3462a
65 changed files with 961 additions and 987 deletions

View File

@ -11,178 +11,152 @@ var outputFile = 'lifestyle',
// feeds.simple_test_feed = require("./config/feed").feed;
// Or define in-line
feeds.simple_test_feed = {
"feed_name" : "feed",
"default_count" : 1,
"no_cdata_fields" : [], // Don't wrap these fields in CDATA tags
"meta" : {
"feed_name": "feed",
"default_count": 1,
"no_cdata_fields": [], // Don't wrap these fields in CDATA tags
"meta": {
"title": "Lifestyle",
"description": "Combined Lifestyle Feed",
'site_url': 'http://pipes.silvrtree.co.uk/lifestyle.xml'
},
"plugins" : ['filter_3_days' ,'fix_images'],
"sources" : [
{
"plugins": ['filter_3_days', 'fix_images', 'fix_scripts'],
"sources": [{
"count" : 100,
"feed_url" : "http://feeds2.feedburner.com/acquire"
"count": 100,
"feed_url": "http://feeds2.feedburner.com/acquire"
},
{
"count" : 100,
"feed_url" : "http://www.fhm.com/category/style/feed"
},
{
"count": 100,
"feed_url": "http://www.fhm.com/category/style/feed"
}, {
"count" : 100,
"feed_url" : "http://www.askmen.com/style.rss"
},
{
"count": 100,
"feed_url": "http://www.askmen.com/style.rss"
}, {
"count" : 100,
"feed_url" : "http://www.askmen.com/fine_living.rss"
},
{
"count": 100,
"feed_url": "http://www.askmen.com/fine_living.rss"
}, {
"count" : 100,
"feed_url" : "http://www.askmen.com/money.rss"
}/*,
{
"count": 100,
"feed_url": "http://www.askmen.com/money.rss"
}
/*,
{
"count" : 100,
"feed_url" : "http://abetterguy.com/feed/"
}*/,
"count" : 100,
"feed_url" : "http://abetterguy.com/feed/"
}*/
,
{
"count" : 100,
"feed_url" : "http://www.dmarge.com/feed"
},
{
"count": 100,
"feed_url": "http://www.dmarge.com/feed"
}, {
"count" : 100,
"feed_url" : "http://www.esquire.com/rss/all.xml"
},
{
"count": 100,
"feed_url": "http://www.esquire.com/rss/all.xml"
}, {
"count" : 100,
"feed_url" : "http://feeds.feedburner.com/gearpatrol"
"count": 100,
"feed_url": "http://feeds.feedburner.com/gearpatrol"
},
{
"count" : 100,
"feed_url" : "http://www.getkempt.com/rss.xml"
},
{
"count": 100,
"feed_url": "http://www.getkempt.com/rss.xml"
}, {
"count" : 100,
"feed_url" : "http://feeds.feedburner.com/kinowear"
},
{
"count": 100,
"feed_url": "http://feeds.feedburner.com/kinowear"
}, {
"count" : 100,
"feed_url" : "http://www.mensfitness.co.uk/feeds/all"
},
{
"count": 100,
"feed_url": "http://www.mensfitness.co.uk/feeds/all"
}, {
"count" : 100,
"feed_url" : "http://feeds.feedburner.com/TheArtOfManliness"
},
{
"count": 100,
"feed_url": "http://feeds.feedburner.com/TheArtOfManliness"
}, {
"count" : 100,
"feed_url" : "http://www.trunkclub.com/blog/feed"
},
{
"count": 100,
"feed_url": "http://www.trunkclub.com/blog/feed"
}, {
"count" : 100,
"feed_url" : "http://feeds.feedburner.com/acontinuouslean/"
},
{
"count": 100,
"feed_url": "http://feeds.feedburner.com/acontinuouslean/"
}, {
"count" : 100,
"feed_url" : "http://www.fashionbeans.com/feed/"
},
{
"count": 100,
"feed_url": "http://www.fashionbeans.com/feed/"
}, {
"count" : 100,
"feed_url" : "http://feeds.feedburner.com/MensFlair?format=xml"
},
{
"count": 100,
"feed_url": "http://feeds.feedburner.com/MensFlair?format=xml"
}, {
"count" : 100,
"feed_url" : "http://feeds.feedburner.com/TheChoosyBeggar"
},
{
"count": 100,
"feed_url": "http://feeds.feedburner.com/TheChoosyBeggar"
}, {
"count" : 100,
"feed_url" : "http://www.valetmag.com/distribution/rss_all.xml"
},
{
"count": 100,
"feed_url": "http://www.valetmag.com/distribution/rss_all.xml"
}, {
"count" : 100,
"feed_url" : "http://putthison.com/rss"
},
{
"count": 100,
"feed_url": "http://putthison.com/rss"
}, {
"count" : 100,
"feed_url" : "http://dieworkwear.com/rss"
},
{
"count": 100,
"feed_url": "http://dieworkwear.com/rss"
}, {
"count" : 100,
"feed_url" : "http://menofhabit.tumblr.com/rss"
},
{
"count": 100,
"feed_url": "http://menofhabit.tumblr.com/rss"
}, {
"count" : 100,
"feed_url" : "http://www.hodinkee.com/blog/rss.xml"
},
{
"count": 100,
"feed_url": "http://www.hodinkee.com/blog/rss.xml"
}, {
"count" : 100,
"feed_url" : "http://www.gq-magazine.co.uk/RSS"
},
{
"count": 100,
"feed_url": "http://www.gq-magazine.co.uk/RSS"
}, {
"count" : 100,
"feed_url" : "http://feeds.feedburner.com/firstinwatches"
},
{
"count": 100,
"feed_url": "http://feeds.feedburner.com/firstinwatches"
}, {
"count" : 100,
"feed_url" : "http://feeds2.feedburner.com/primermagazine/otzU"
},
{
"count": 100,
"feed_url": "http://feeds2.feedburner.com/primermagazine/otzU"
}, {
"count" : 100,
"feed_url" : "http://chrisguillebeau.com/3x5/feed/"
},
{
"count": 100,
"feed_url": "http://chrisguillebeau.com/3x5/feed/"
}, {
"count" : 100,
"feed_url" : "http://feeds.feedburner.com/TheAwesomer"
},
{
"count": 100,
"feed_url": "http://feeds.feedburner.com/TheAwesomer"
}, {
"count" : 100,
"feed_url" : "http://bondclothes.blogspot.com/feeds/posts/default"
},
{
"count": 100,
"feed_url": "http://bondclothes.blogspot.com/feeds/posts/default"
}, {
"count" : 100,
"feed_url" : "http://affordablebond007.tumblr.com/rss"
},
{
"count": 100,
"feed_url": "http://affordablebond007.tumblr.com/rss"
}, {
"count" : 100,
"feed_url" : "http://thecozyspace.tumblr.com/rss"
},
{
"count": 100,
"feed_url": "http://thecozyspace.tumblr.com/rss"
}, {
"count" : 100,
"feed_url" : "http://fourpins.com/feed/"
"count": 100,
"feed_url": "http://fourpins.com/feed/"
}
]
};
@ -192,14 +166,14 @@ var braider_options = {
date_sort_order: "desc", // Newest first
log_level: "warn",
dedupe_fields: ['link', 'guid'],
plugins_directories : [__dirname + "/plugins/"]
plugins_directories: [__dirname + "/plugins/"]
};
var rss_braider = RssBraider.createClient(braider_options);
// Override logging level (debug, info, warn, err, off)
rss_braider.logger.level('off');
rss_braider.processFeed('simple_test_feed', 'json', function (err, data) {
rss_braider.processFeed('simple_test_feed', 'json', function(err, data) {
if (err) {
return console.log(err);
}
@ -207,7 +181,7 @@ rss_braider.processFeed('simple_test_feed', 'json', function (err, data) {
var j = JSON.parse(data);
var ejsOutput = ejs.compile(str)(j);
fs.writeFile(__dirname + "/html/" + outputFile + ".json", data, function (err) {
fs.writeFile(__dirname + "/html/" + outputFile + ".json", data, function(err) {
if (err) {
return console.log(err);
}
@ -215,7 +189,7 @@ rss_braider.processFeed('simple_test_feed', 'json', function (err, data) {
console.log("The file was saved!");
});
fs.writeFile(__dirname + "/html/" + outputFile + ".html", ejsOutput, function (err) {
fs.writeFile(__dirname + "/html/" + outputFile + ".html", ejsOutput, function(err) {
if (err) {
return console.log(err);
@ -237,4 +211,3 @@ rss_braider.processFeed('simple_test_feed', 'rss', function (err, data) {
});
});
*/

View File

@ -1,20 +1,20 @@
.idea
*.iml
npm-debug.log
dump.rdb
node_modules
components
build
results.tap
results.xml
npm-shrinkwrap.json
config.json
.DS_Store
*/.DS_Store
*/*/.DS_Store
._*
*/._*
*/*/._*
coverage.*
lib-cov
.idea
*.iml
npm-debug.log
dump.rdb
node_modules
components
build
results.tap
results.xml
npm-shrinkwrap.json
config.json
.DS_Store
*/.DS_Store
*/*/.DS_Store
._*
*/._*
*/*/._*
coverage.*
lib-cov

10
node_modules/rss-braider/node_modules/request/node_modules/hawk/.travis.yml generated vendored Executable file → Normal file
View File

@ -1,5 +1,5 @@
language: node_js
node_js:
- 0.10
language: node_js
node_js:
- 0.10

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/LICENSE generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/README.md generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/example/usage.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/images/hawk.png generated vendored Executable file → Normal file
View File

Before

Width:  |  Height:  |  Size: 6.8 KiB

After

Width:  |  Height:  |  Size: 6.8 KiB

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/images/logo.png generated vendored Executable file → Normal file
View File

Before

Width:  |  Height:  |  Size: 70 KiB

After

Width:  |  Height:  |  Size: 70 KiB

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/lib/browser.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/lib/client.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/lib/crypto.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/lib/index.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/lib/server.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/lib/utils.js generated vendored Executable file → Normal file
View File

View File

View File

View File

View File

Before

Width:  |  Height:  |  Size: 29 KiB

After

Width:  |  Height:  |  Size: 29 KiB

View File

View File

View File

View File

View File

View File

View File

View File

View File

View File

View File

View File

Before

Width:  |  Height:  |  Size: 37 KiB

After

Width:  |  Height:  |  Size: 37 KiB

View File

View File

View File

@ -612,12 +612,12 @@ exports.reach = function (obj, chain, options) {
};
exports.reachTemplate = function (obj, template, options) {
return template.replace(/{([^}]+)}/g, function ($0, chain) {
exports.reachTemplate = function (obj, template, options) {
var value = exports.reach(obj, chain, options);
return (value === undefined || value === null ? '' : value);
return template.replace(/{([^}]+)}/g, function ($0, chain) {
var value = exports.reach(obj, chain, options);
return (value === undefined || value === null ? '' : value);
});
};

View File

File diff suppressed because it is too large Load Diff

View File

View File

View File

View File

View File

View File

View File

View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/test/browser.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/test/client.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/test/crypto.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/test/index.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/test/readme.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/test/server.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/test/uri.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/test/utils.js generated vendored Executable file → Normal file
View File

View File

0
node_modules/rss-braider/node_modules/request/node_modules/node-uuid/bin/uuid generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/release.sh generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/rss/lib/index.js generated vendored Executable file → Normal file
View File

View File

@ -20,50 +20,40 @@ feeds.simple_test_feed = {
"description": "Combined Paleo Feed",
'site_url': 'http://pipes.silvrtree.co.uk/paleo.xml'
},
"plugins" : ['filter_3_days',"fix_images" ],
"sources": [
{
/* "name" : "JobServe",*/
"count": 100,
"feed_url": "http://feeds.feedburner.com/PaleoPlan"
},
{
/* "name" : "JobServe",*/
"count": 100,
"feed_url": "http://nomnompaleo.com/rss"
},
{
/* "name" : "JobServe",*/
"count": 100,
"feed_url": "http://thepaleodiet.com/feed/"
},
{
/* "name" : "JobServe",*/
"count": 100,
"feed_url": "http://paleoleap.com/feed/"
},
{
/* "name" : "JobServe",*/
"count": 100,
"feed_url": "http://ultimatepaleoguide.com/feed/"
},
{
/* "name" : "JobServe",*/
"count": 100,
"feed_url": "http://elanaspantry.com/feed/"
}
,
{
/* "name" : "JobServe",*/
"count": 100,
"feed_url": "http://feeds.feedburner.com/Paleomg-PaleoRecipes"
},
{
/* "name" : "JobServe",*/
"count": 100,
"feed_url": "http://civilizedcavemancooking.com/feed/"
}
]
"plugins": ['filter_3_days', "fix_images", 'fix_scripts'],
"sources": [{
/* "name" : "JobServe",*/
"count": 100,
"feed_url": "http://feeds.feedburner.com/PaleoPlan"
}, {
/* "name" : "JobServe",*/
"count": 100,
"feed_url": "http://nomnompaleo.com/rss"
}, {
/* "name" : "JobServe",*/
"count": 100,
"feed_url": "http://thepaleodiet.com/feed/"
}, {
/* "name" : "JobServe",*/
"count": 100,
"feed_url": "http://paleoleap.com/feed/"
}, {
/* "name" : "JobServe",*/
"count": 100,
"feed_url": "http://ultimatepaleoguide.com/feed/"
}, {
/* "name" : "JobServe",*/
"count": 100,
"feed_url": "http://elanaspantry.com/feed/"
}, {
/* "name" : "JobServe",*/
"count": 100,
"feed_url": "http://feeds.feedburner.com/Paleomg-PaleoRecipes"
}, {
/* "name" : "JobServe",*/
"count": 100,
"feed_url": "http://civilizedcavemancooking.com/feed/"
}]
};
var braider_options = {
feeds: feeds,
@ -71,14 +61,14 @@ var braider_options = {
date_sort_order: "desc", // Newest first
log_level: "warn",
dedupe_fields: ['guid'],
plugins_directories : [__dirname + "/plugins/"]
plugins_directories: [__dirname + "/plugins/"]
};
var rss_braider = RssBraider.createClient(braider_options);
// Override logging level (debug, info, warn, err, off)
rss_braider.logger.level('off');
rss_braider.processFeed('simple_test_feed', 'json', function (err, data) {
rss_braider.processFeed('simple_test_feed', 'json', function(err, data) {
if (err) {
return console.log(err);
}
@ -86,7 +76,7 @@ rss_braider.processFeed('simple_test_feed', 'json', function (err, data) {
var j = JSON.parse(data);
var ejsOutput = ejs.compile(str)(j);
fs.writeFile(__dirname + "/html/" + outputFile + ".json", data, function (err) {
fs.writeFile(__dirname + "/html/" + outputFile + ".json", data, function(err) {
if (err) {
return console.log(err);
}

7
plugins/fix_scripts.js Normal file
View File

@ -0,0 +1,7 @@
module.exports = function (item, itemOptions, source) {
var match = itemOptions.description.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi,'');
itemOptions.description = match;
return itemOptions;
};

View File

@ -20,7 +20,7 @@ feeds.simple_test_feed = {
"description": "Combined Tech Feed",
'site_url': 'http://pipes.silvrtree.co.uk/tech.xml'
},
"plugins": ['filter_3_days', 'fix_images'],
"plugins": ['filter_3_days', 'fix_images', 'fix_scripts'],
"sources": [{
"count": 100,
"feed_url": "http://feeds.feedburner.com/lenovoblogs/designmatters"
@ -69,7 +69,11 @@ feeds.simple_test_feed = {
}, {
"count": 100,
"feed_url": "https://medium.com/feed/google-developers"
}]
},
{
"count":100,
"feed_url":"http://feedproxy.google.com/typepad/alleyinsider/silicon_alley_insider"
}]
};
var braider_options = {
feeds: feeds,