added a new script filter and no idea wtf is up with node

This commit is contained in:
Martin Donnelly 2015-08-14 10:55:39 +01:00
parent d73a93ec9e
commit ab44d3462a
65 changed files with 961 additions and 987 deletions

View File

@ -11,178 +11,152 @@ var outputFile = 'lifestyle',
// feeds.simple_test_feed = require("./config/feed").feed; // feeds.simple_test_feed = require("./config/feed").feed;
// Or define in-line // Or define in-line
feeds.simple_test_feed = { feeds.simple_test_feed = {
"feed_name" : "feed", "feed_name": "feed",
"default_count" : 1, "default_count": 1,
"no_cdata_fields" : [], // Don't wrap these fields in CDATA tags "no_cdata_fields": [], // Don't wrap these fields in CDATA tags
"meta" : { "meta": {
"title": "Lifestyle", "title": "Lifestyle",
"description": "Combined Lifestyle Feed", "description": "Combined Lifestyle Feed",
'site_url': 'http://pipes.silvrtree.co.uk/lifestyle.xml' 'site_url': 'http://pipes.silvrtree.co.uk/lifestyle.xml'
}, },
"plugins" : ['filter_3_days' ,'fix_images'], "plugins": ['filter_3_days', 'fix_images', 'fix_scripts'],
"sources" : [ "sources": [{
{
"count" : 100, "count": 100,
"feed_url" : "http://feeds2.feedburner.com/acquire" "feed_url": "http://feeds2.feedburner.com/acquire"
}, },
{ {
"count" : 100, "count": 100,
"feed_url" : "http://www.fhm.com/category/style/feed" "feed_url": "http://www.fhm.com/category/style/feed"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://www.askmen.com/style.rss" "feed_url": "http://www.askmen.com/style.rss"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://www.askmen.com/fine_living.rss" "feed_url": "http://www.askmen.com/fine_living.rss"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://www.askmen.com/money.rss" "feed_url": "http://www.askmen.com/money.rss"
}/*, }
{ /*,
{
"count" : 100, "count" : 100,
"feed_url" : "http://abetterguy.com/feed/" "feed_url" : "http://abetterguy.com/feed/"
}*/, }*/
,
{ {
"count" : 100, "count": 100,
"feed_url" : "http://www.dmarge.com/feed" "feed_url": "http://www.dmarge.com/feed"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://www.esquire.com/rss/all.xml" "feed_url": "http://www.esquire.com/rss/all.xml"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://feeds.feedburner.com/gearpatrol" "feed_url": "http://feeds.feedburner.com/gearpatrol"
}, },
{ {
"count" : 100, "count": 100,
"feed_url" : "http://www.getkempt.com/rss.xml" "feed_url": "http://www.getkempt.com/rss.xml"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://feeds.feedburner.com/kinowear" "feed_url": "http://feeds.feedburner.com/kinowear"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://www.mensfitness.co.uk/feeds/all" "feed_url": "http://www.mensfitness.co.uk/feeds/all"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://feeds.feedburner.com/TheArtOfManliness" "feed_url": "http://feeds.feedburner.com/TheArtOfManliness"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://www.trunkclub.com/blog/feed" "feed_url": "http://www.trunkclub.com/blog/feed"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://feeds.feedburner.com/acontinuouslean/" "feed_url": "http://feeds.feedburner.com/acontinuouslean/"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://www.fashionbeans.com/feed/" "feed_url": "http://www.fashionbeans.com/feed/"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://feeds.feedburner.com/MensFlair?format=xml" "feed_url": "http://feeds.feedburner.com/MensFlair?format=xml"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://feeds.feedburner.com/TheChoosyBeggar" "feed_url": "http://feeds.feedburner.com/TheChoosyBeggar"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://www.valetmag.com/distribution/rss_all.xml" "feed_url": "http://www.valetmag.com/distribution/rss_all.xml"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://putthison.com/rss" "feed_url": "http://putthison.com/rss"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://dieworkwear.com/rss" "feed_url": "http://dieworkwear.com/rss"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://menofhabit.tumblr.com/rss" "feed_url": "http://menofhabit.tumblr.com/rss"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://www.hodinkee.com/blog/rss.xml" "feed_url": "http://www.hodinkee.com/blog/rss.xml"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://www.gq-magazine.co.uk/RSS" "feed_url": "http://www.gq-magazine.co.uk/RSS"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://feeds.feedburner.com/firstinwatches" "feed_url": "http://feeds.feedburner.com/firstinwatches"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://feeds2.feedburner.com/primermagazine/otzU" "feed_url": "http://feeds2.feedburner.com/primermagazine/otzU"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://chrisguillebeau.com/3x5/feed/" "feed_url": "http://chrisguillebeau.com/3x5/feed/"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://feeds.feedburner.com/TheAwesomer" "feed_url": "http://feeds.feedburner.com/TheAwesomer"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://bondclothes.blogspot.com/feeds/posts/default" "feed_url": "http://bondclothes.blogspot.com/feeds/posts/default"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://affordablebond007.tumblr.com/rss" "feed_url": "http://affordablebond007.tumblr.com/rss"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://thecozyspace.tumblr.com/rss" "feed_url": "http://thecozyspace.tumblr.com/rss"
}, }, {
{
"count" : 100, "count": 100,
"feed_url" : "http://fourpins.com/feed/" "feed_url": "http://fourpins.com/feed/"
} }
] ]
}; };
@ -192,14 +166,14 @@ var braider_options = {
date_sort_order: "desc", // Newest first date_sort_order: "desc", // Newest first
log_level: "warn", log_level: "warn",
dedupe_fields: ['link', 'guid'], dedupe_fields: ['link', 'guid'],
plugins_directories : [__dirname + "/plugins/"] plugins_directories: [__dirname + "/plugins/"]
}; };
var rss_braider = RssBraider.createClient(braider_options); var rss_braider = RssBraider.createClient(braider_options);
// Override logging level (debug, info, warn, err, off) // Override logging level (debug, info, warn, err, off)
rss_braider.logger.level('off'); rss_braider.logger.level('off');
rss_braider.processFeed('simple_test_feed', 'json', function (err, data) { rss_braider.processFeed('simple_test_feed', 'json', function(err, data) {
if (err) { if (err) {
return console.log(err); return console.log(err);
} }
@ -207,7 +181,7 @@ rss_braider.processFeed('simple_test_feed', 'json', function (err, data) {
var j = JSON.parse(data); var j = JSON.parse(data);
var ejsOutput = ejs.compile(str)(j); var ejsOutput = ejs.compile(str)(j);
fs.writeFile(__dirname + "/html/" + outputFile + ".json", data, function (err) { fs.writeFile(__dirname + "/html/" + outputFile + ".json", data, function(err) {
if (err) { if (err) {
return console.log(err); return console.log(err);
} }
@ -215,7 +189,7 @@ rss_braider.processFeed('simple_test_feed', 'json', function (err, data) {
console.log("The file was saved!"); console.log("The file was saved!");
}); });
fs.writeFile(__dirname + "/html/" + outputFile + ".html", ejsOutput, function (err) { fs.writeFile(__dirname + "/html/" + outputFile + ".html", ejsOutput, function(err) {
if (err) { if (err) {
return console.log(err); return console.log(err);
@ -237,4 +211,3 @@ rss_braider.processFeed('simple_test_feed', 'rss', function (err, data) {
}); });
}); });
*/ */

View File

@ -1,20 +1,20 @@
.idea .idea
*.iml *.iml
npm-debug.log npm-debug.log
dump.rdb dump.rdb
node_modules node_modules
components components
build build
results.tap results.tap
results.xml results.xml
npm-shrinkwrap.json npm-shrinkwrap.json
config.json config.json
.DS_Store .DS_Store
*/.DS_Store */.DS_Store
*/*/.DS_Store */*/.DS_Store
._* ._*
*/._* */._*
*/*/._* */*/._*
coverage.* coverage.*
lib-cov lib-cov

10
node_modules/rss-braider/node_modules/request/node_modules/hawk/.travis.yml generated vendored Executable file → Normal file
View File

@ -1,5 +1,5 @@
language: node_js language: node_js
node_js: node_js:
- 0.10 - 0.10

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/LICENSE generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/README.md generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/example/usage.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/images/hawk.png generated vendored Executable file → Normal file
View File

Before

Width:  |  Height:  |  Size: 6.8 KiB

After

Width:  |  Height:  |  Size: 6.8 KiB

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/images/logo.png generated vendored Executable file → Normal file
View File

Before

Width:  |  Height:  |  Size: 70 KiB

After

Width:  |  Height:  |  Size: 70 KiB

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/lib/browser.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/lib/client.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/lib/crypto.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/lib/index.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/lib/server.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/lib/utils.js generated vendored Executable file → Normal file
View File

View File

View File

View File

View File

Before

Width:  |  Height:  |  Size: 29 KiB

After

Width:  |  Height:  |  Size: 29 KiB

View File

View File

View File

View File

View File

View File

View File

View File

View File

View File

View File

View File

Before

Width:  |  Height:  |  Size: 37 KiB

After

Width:  |  Height:  |  Size: 37 KiB

View File

View File

View File

@ -612,12 +612,12 @@ exports.reach = function (obj, chain, options) {
}; };
exports.reachTemplate = function (obj, template, options) { exports.reachTemplate = function (obj, template, options) {
return template.replace(/{([^}]+)}/g, function ($0, chain) {
var value = exports.reach(obj, chain, options); return template.replace(/{([^}]+)}/g, function ($0, chain) {
return (value === undefined || value === null ? '' : value);
var value = exports.reach(obj, chain, options);
return (value === undefined || value === null ? '' : value);
}); });
}; };

View File

File diff suppressed because it is too large Load Diff

View File

View File

View File

View File

View File

View File

View File

View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/test/browser.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/test/client.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/test/crypto.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/test/index.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/test/readme.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/test/server.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/test/uri.js generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/node_modules/hawk/test/utils.js generated vendored Executable file → Normal file
View File

View File

0
node_modules/rss-braider/node_modules/request/node_modules/node-uuid/bin/uuid generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/request/release.sh generated vendored Executable file → Normal file
View File

0
node_modules/rss-braider/node_modules/rss/lib/index.js generated vendored Executable file → Normal file
View File

View File

@ -20,50 +20,40 @@ feeds.simple_test_feed = {
"description": "Combined Paleo Feed", "description": "Combined Paleo Feed",
'site_url': 'http://pipes.silvrtree.co.uk/paleo.xml' 'site_url': 'http://pipes.silvrtree.co.uk/paleo.xml'
}, },
"plugins" : ['filter_3_days',"fix_images" ], "plugins": ['filter_3_days', "fix_images", 'fix_scripts'],
"sources": [ "sources": [{
{ /* "name" : "JobServe",*/
/* "name" : "JobServe",*/ "count": 100,
"count": 100, "feed_url": "http://feeds.feedburner.com/PaleoPlan"
"feed_url": "http://feeds.feedburner.com/PaleoPlan" }, {
}, /* "name" : "JobServe",*/
{ "count": 100,
/* "name" : "JobServe",*/ "feed_url": "http://nomnompaleo.com/rss"
"count": 100, }, {
"feed_url": "http://nomnompaleo.com/rss" /* "name" : "JobServe",*/
}, "count": 100,
{ "feed_url": "http://thepaleodiet.com/feed/"
/* "name" : "JobServe",*/ }, {
"count": 100, /* "name" : "JobServe",*/
"feed_url": "http://thepaleodiet.com/feed/" "count": 100,
}, "feed_url": "http://paleoleap.com/feed/"
{ }, {
/* "name" : "JobServe",*/ /* "name" : "JobServe",*/
"count": 100, "count": 100,
"feed_url": "http://paleoleap.com/feed/" "feed_url": "http://ultimatepaleoguide.com/feed/"
}, }, {
{ /* "name" : "JobServe",*/
/* "name" : "JobServe",*/ "count": 100,
"count": 100, "feed_url": "http://elanaspantry.com/feed/"
"feed_url": "http://ultimatepaleoguide.com/feed/" }, {
}, /* "name" : "JobServe",*/
{ "count": 100,
/* "name" : "JobServe",*/ "feed_url": "http://feeds.feedburner.com/Paleomg-PaleoRecipes"
"count": 100, }, {
"feed_url": "http://elanaspantry.com/feed/" /* "name" : "JobServe",*/
} "count": 100,
, "feed_url": "http://civilizedcavemancooking.com/feed/"
{ }]
/* "name" : "JobServe",*/
"count": 100,
"feed_url": "http://feeds.feedburner.com/Paleomg-PaleoRecipes"
},
{
/* "name" : "JobServe",*/
"count": 100,
"feed_url": "http://civilizedcavemancooking.com/feed/"
}
]
}; };
var braider_options = { var braider_options = {
feeds: feeds, feeds: feeds,
@ -71,14 +61,14 @@ var braider_options = {
date_sort_order: "desc", // Newest first date_sort_order: "desc", // Newest first
log_level: "warn", log_level: "warn",
dedupe_fields: ['guid'], dedupe_fields: ['guid'],
plugins_directories : [__dirname + "/plugins/"] plugins_directories: [__dirname + "/plugins/"]
}; };
var rss_braider = RssBraider.createClient(braider_options); var rss_braider = RssBraider.createClient(braider_options);
// Override logging level (debug, info, warn, err, off) // Override logging level (debug, info, warn, err, off)
rss_braider.logger.level('off'); rss_braider.logger.level('off');
rss_braider.processFeed('simple_test_feed', 'json', function (err, data) { rss_braider.processFeed('simple_test_feed', 'json', function(err, data) {
if (err) { if (err) {
return console.log(err); return console.log(err);
} }
@ -86,7 +76,7 @@ rss_braider.processFeed('simple_test_feed', 'json', function (err, data) {
var j = JSON.parse(data); var j = JSON.parse(data);
var ejsOutput = ejs.compile(str)(j); var ejsOutput = ejs.compile(str)(j);
fs.writeFile(__dirname + "/html/" + outputFile + ".json", data, function (err) { fs.writeFile(__dirname + "/html/" + outputFile + ".json", data, function(err) {
if (err) { if (err) {
return console.log(err); return console.log(err);
} }

7
plugins/fix_scripts.js Normal file
View File

@ -0,0 +1,7 @@
module.exports = function (item, itemOptions, source) {
var match = itemOptions.description.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi,'');
itemOptions.description = match;
return itemOptions;
};

View File

@ -20,7 +20,7 @@ feeds.simple_test_feed = {
"description": "Combined Tech Feed", "description": "Combined Tech Feed",
'site_url': 'http://pipes.silvrtree.co.uk/tech.xml' 'site_url': 'http://pipes.silvrtree.co.uk/tech.xml'
}, },
"plugins": ['filter_3_days', 'fix_images'], "plugins": ['filter_3_days', 'fix_images', 'fix_scripts'],
"sources": [{ "sources": [{
"count": 100, "count": 100,
"feed_url": "http://feeds.feedburner.com/lenovoblogs/designmatters" "feed_url": "http://feeds.feedburner.com/lenovoblogs/designmatters"
@ -69,7 +69,11 @@ feeds.simple_test_feed = {
}, { }, {
"count": 100, "count": 100,
"feed_url": "https://medium.com/feed/google-developers" "feed_url": "https://medium.com/feed/google-developers"
}] },
{
"count":100,
"feed_url":"http://feedproxy.google.com/typepad/alleyinsider/silicon_alley_insider"
}]
}; };
var braider_options = { var braider_options = {
feeds: feeds, feeds: feeds,