150 lines
5.9 KiB
JavaScript
150 lines
5.9 KiB
JavaScript
var outputFile = 'news', RssBraider = require('rss-braider'), fs = require('fs'), ejs = require(
|
|
'ejs'), read = require('fs').readFileSync, join = require('path').join, str = read(
|
|
join(__dirname, '/templates/rss.ejs'),
|
|
'utf8'), feeds = {};
|
|
|
|
var argv = require('yargs').argv;
|
|
|
|
var log4js = require('log4js');
|
|
var logger = log4js.getLogger();
|
|
|
|
// Pull feeds from config files: ,
|
|
// feeds.simple_test_feed = require("./config/feed").feed;
|
|
// Or define in-line
|
|
|
|
const f = ["http://qz.com/feed/",
|
|
"http://feeds2.feedburner.com/businessinsider",
|
|
"http://feeds.reuters.com/reuters/UKTopNews",
|
|
"http://feeds.reuters.com/reuters/UKdomesticNews",
|
|
"http://feeds.reuters.com/reuters/UKWorldNews",
|
|
"http://feeds.reuters.com/reuters/topNews",
|
|
"http://feeds.reuters.com/Reuters/worldNews",
|
|
"http://hosted2.ap.org/atom/APDEFAULT/cae69a7523db45408eeb2b3a98c0c9c5",
|
|
"http://www.cityam.com/feeds/main.xml",
|
|
"http://www.londonlovesbusiness.com/navrss?navsectioncode=121",
|
|
"http://www.ibtimes.co.uk/rss/uk",
|
|
"http://feeds.bbci.co.uk/news/rss.xml?edition=uk",
|
|
"http://feeds.bbci.co.uk/news/scotland/rss.xml?edition=uk",
|
|
"http://news.google.com/?output=rss",
|
|
"http://feeds.feedburner.com/TheAtlantic",
|
|
"http://www.telegraph.co.uk/news/rss.xml",
|
|
"http://www.guardian.co.uk/rssfeed/0,,1,00.xml",
|
|
"http://feeds.marketwatch.com/marketwatch/topstories",
|
|
"http://feeds.feedburner.com/wingsoverscotland/FgKv",
|
|
"http://bellacaledonia.org.uk/feed/",
|
|
"http://www.theweek.co.uk/feeds/all",
|
|
"http://fivethirtyeight.com/all/feed"];
|
|
|
|
|
|
|
|
feeds.simple_test_feed = {
|
|
'feed_name': 'feed',
|
|
'default_count': 1,
|
|
'no_cdata_fields': ['description'], // Don't wrap these fields in CDATA tags
|
|
'meta': {
|
|
'title': 'News',
|
|
'description': 'Combined News Feed',
|
|
'site_url': 'http://pipes.silvrtree.co.uk/news.xml'
|
|
},
|
|
'plugins': [
|
|
'filter_3_days', 'fix_images', 'fix_scripts'
|
|
],
|
|
'custom_namespaces': {
|
|
'content': 'http://purl.org/rss/1.0/modules/content/',
|
|
'slash': 'http://purl.org/rss/1.0/modules/slash/',
|
|
'media': 'http://search.yahoo.com/mrss/',
|
|
'ev': 'http://purl.org/rss/2.0/modules/event/',
|
|
'sy': 'http://purl.org/rss/1.0/modules/syndication/',
|
|
'wfw': 'http://wellformedweb.org/CommentAPI/',
|
|
'dc': 'http://purl.org/dc/elements/1.1/',
|
|
'atom': 'http://www.w3.org/2005/Atom'
|
|
},
|
|
'sources': [{'count': 100, 'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Fqz.com%2Ffeed%2F'}, {
|
|
'count': 100,
|
|
'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Ffeeds2.feedburner.com%2Fbusinessinsider'
|
|
}, {'count': 100, 'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Ffeeds.reuters.com%2Freuters%2FUKTopNews'}, {
|
|
'count': 100,
|
|
'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Ffeeds.reuters.com%2Freuters%2FUKdomesticNews'
|
|
}, {'count': 100, 'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Ffeeds.reuters.com%2Freuters%2FUKWorldNews'}, {
|
|
'count': 100,
|
|
'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Ffeeds.reuters.com%2Freuters%2FtopNews'
|
|
}, {'count': 100, 'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Ffeeds.reuters.com%2FReuters%2FworldNews'}, {
|
|
'count': 100,
|
|
'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Fhosted2.ap.org%2Fatom%2FAPDEFAULT%2Fcae69a7523db45408eeb2b3a98c0c9c5'
|
|
}, {'count': 100, 'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Fwww.cityam.com%2Ffeeds%2Fmain.xml'}, {
|
|
'count': 100,
|
|
'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Fwww.londonlovesbusiness.com%2Fnavrss%3Fnavsectioncode%3D121'
|
|
}, {'count': 100, 'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Fwww.ibtimes.co.uk%2Frss%2Fuk'}, {
|
|
'count': 100,
|
|
'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Ffeeds.bbci.co.uk%2Fnews%2Frss.xml%3Fedition%3Duk'
|
|
}, {
|
|
'count': 100,
|
|
'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Ffeeds.bbci.co.uk%2Fnews%2Fscotland%2Frss.xml%3Fedition%3Duk'
|
|
}, {'count': 100, 'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Fnews.google.com%2F%3Foutput%3Drss'}, {
|
|
'count': 100,
|
|
'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Ffeeds.feedburner.com%2FTheAtlantic'
|
|
}, {'count': 100, 'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Fwww.telegraph.co.uk%2Fnews%2Frss.xml'}, {
|
|
'count': 100,
|
|
'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Fwww.guardian.co.uk%2Frssfeed%2F0%2C%2C1%2C00.xml'
|
|
}, {'count': 100, 'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Ffeeds.marketwatch.com%2Fmarketwatch%2Ftopstories%2F'}, {
|
|
'count': 100,
|
|
'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Ffeeds.feedburner.com%2Fwingsoverscotland%2FFgKv'
|
|
}, {'count': 100, 'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Fbellacaledonia.org.uk%2Ffeed%2F'}, {
|
|
'count': 100,
|
|
'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Fwww.theweek.co.uk%2Ffeeds%2Fall'
|
|
}, {'count': 100, 'feed_url': 'http://45.33.114.116:8080/http%3A%2F%2Ffivethirtyeight.com%2Fall%2Ffeed'}]
|
|
};
|
|
|
|
|
|
var braider_options = {
|
|
feeds: feeds,
|
|
indent: ' ',
|
|
date_sort_order: 'desc', // Newest first
|
|
log_level: 'warn',
|
|
dedupe_fields: ['link', 'guid'],
|
|
plugins_directories: [__dirname + '/plugins/']
|
|
};
|
|
var rss_braider = RssBraider.createClient(braider_options);
|
|
|
|
// Override logging level (debug, info, warn, err, off)
|
|
//rss_braider.logger.level('off');
|
|
|
|
rss_braider.processFeed('simple_test_feed', 'json', function (err, data) {
|
|
console.log('A');
|
|
if (err) {
|
|
logger.error(err);
|
|
return err;
|
|
}
|
|
|
|
console.log('B');
|
|
|
|
var j = JSON.parse(data);
|
|
|
|
console.log('C');
|
|
var ejsOutput = ejs.compile(str)(j);
|
|
|
|
console.log('D');
|
|
fs.writeFile(__dirname + '/dist/' + outputFile + '.html',
|
|
ejsOutput,
|
|
function (err) {
|
|
|
|
if (err) {
|
|
logger.error(err);
|
|
return console.error(err);
|
|
}
|
|
|
|
console.log('The file was saved!');
|
|
});
|
|
|
|
fs.writeFile(__dirname + '/dist/' + outputFile + '.json',
|
|
data,
|
|
function (err) {
|
|
if (err) {
|
|
logger.error(err);
|
|
return console.error(err);
|
|
}
|
|
|
|
console.log('The file was saved!');
|
|
});
|
|
});
|