From dedd83a446116eb10416b59bba8bd9cb68b303a7 Mon Sep 17 00:00:00 2001 From: Martin Donnelly Date: Thu, 9 Aug 2018 17:05:59 +0100 Subject: [PATCH] tighter jobs-special --- .idea/workspace.xml | 293 +++++++++++++++++++++-------------------- plugins/filter_show.js | 5 + test.js | 231 ++++++++++---------------------- 3 files changed, 221 insertions(+), 308 deletions(-) create mode 100644 plugins/filter_show.js diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 37cfeb3..eb7a47f 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,14 +2,9 @@ - + - - - - - - + @@ -34,8 +29,8 @@ - - + + @@ -49,20 +44,20 @@ - - + + - - + + - - + + - - + + @@ -70,53 +65,53 @@ - - + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + @@ -133,7 +128,6 @@ - html processJson worki techno @@ -163,6 +157,7 @@ fs_writeFile jobs mensf + http%3A%2F%2Fwww.purelyit.co.uk%2Fcgi-bin%2Fadvsearch%3Frss_feed%3D1%26daysback%3D1%26jbe_id%3D3325606 @@ -185,8 +180,6 @@ @@ -611,21 +606,7 @@ - - - - 1500371543772 - - - 1500371811918 - 1500371930584 @@ -956,22 +937,36 @@ - - - - + + - + @@ -992,29 +987,29 @@ - - - - - - - - + + + + + + + + - - - + - - + - + + + + - - - + + + @@ -1071,7 +1066,6 @@ @@ -1123,15 +1118,6 @@ - - - - - - - - - @@ -1225,13 +1211,6 @@ - - - - - - - @@ -1309,13 +1288,6 @@ - - - - - - - @@ -1409,13 +1381,6 @@ - - - - - - - @@ -1423,13 +1388,6 @@ - - - - - - - @@ -1444,17 +1402,66 @@ - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + diff --git a/plugins/filter_show.js b/plugins/filter_show.js new file mode 100644 index 0000000..3ac2af5 --- /dev/null +++ b/plugins/filter_show.js @@ -0,0 +1,5 @@ +module.exports = function (item, itemOptions, source) { + console.log(itemOptions); + + return itemOptions; +}; diff --git a/test.js b/test.js index e0fafb3..66ab3a0 100644 --- a/test.js +++ b/test.js @@ -1,182 +1,83 @@ -var outputFile = 'news', - RssBraider = require('rss-braider'), - fs = require('fs'), - ejs = require('ejs'), - read = require('fs').readFileSync, - join = require('path').join, - str = read(join(__dirname, '/templates/rss.ejs'), 'utf8'), - feeds = {}; +var outputFile = 'jobs-local', + RssBraider = require('rss-braider'), + fs = require('fs'), + ejs = require('ejs'), + read = require('fs').readFileSync, + join = require('path').join, + str = read(join(__dirname, '/templates/rss.ejs'), 'utf8'), + feeds = {}; -var log4js = require('log4js'); -var logger = log4js.getLogger(); - - -// Pull feeds from config files: , +// Pull feeds from config files: // feeds.simple_test_feed = require("./config/feed").feed; // Or define in-line - feeds.simple_test_feed = { - "feed_name": "feed", - "default_count": 1, - "no_cdata_fields": [], - // Don't wrap these fields in CDATA tags - "meta": { - "title": "News", - "description": "Combined News Feed", - 'site_url': 'http://pipes.silvrtree.co.uk/news.xml' - }, - "plugins": [ - 'filter_3_days', - 'fix_images', - 'fix_scripts' + 'feed_name': 'feed', + 'default_count': 1, + 'no_cdata_fields': [], // Don't wrap these fields in CDATA tags + 'plugins': [ + 'filter_show', + 'filter_location', + 'filter_reject', + 'filter_md_jobs', + 'filter_last_week' ], - "sources": [ - { - "count": 100, - "feed_url": "http://qz.com/feed/" - }, /* - { - "count": 100, - "feed_url": "http://feeds2.feedburner.com/businessinsider" - }, - /* - { - "count": 100, - "feed_url": "http://feeds.reuters.com/reuters/UKTopNews" - }, - { - "count": 100, - "feed_url": "http://feeds.reuters.com/reuters/UKdomesticNews" - }, - { - "count": 100, - "feed_url": "http://feeds.reuters.com/reuters/UKWorldNews" - } -, - { - "count": 100, - "feed_url": "http://feeds.reuters.com/reuters/topNews" - } - , - { - "count": 100, - "feed_url": "http://feeds.reuters.com/Reuters/worldNews" - } + 'meta': { + 'title': 'Jobs', + 'description': 'Combined Jobs Feed', + 'site_url': 'http://pipes.silvrtree.co.uk/jobs-local.xml' + }, + 'sources': [{ + 'count': 100, + 'feed_url': 'http://45.33.114.116:8080/https%3A%2F%2Fwww.jobserve.com%2FMySearch%2FBAEBF3BDF82B8FEF.rss' + }] +}; - , - { - "count": 100, - "feed_url": "http://feeds.reuters.com/reuters/globalmarketsNews" - } - , /* - { - "count": 100, - "feed_url": "http://hosted2.ap.org/atom/APDEFAULT/cae69a7523db45408eeb2b3a98c0c9c5" - }, /* - { - "count": 100, - "feed_url": "http://www.cityam.com/feeds/main.xml" - }, /* - { - "count": 100, - "feed_url": "http://www.londonlovesbusiness.com/navrss?navsectioncode=121" - }, /* - { - "count": 100, - "feed_url": "http://www.ibtimes.co.uk/rss/uk" - }, /* - { - "count": 100, - "feed_url": "http://feeds.bbci.co.uk/news/rss.xml?edition=uk" - }, - { - "count": 100, - "feed_url": "http://feeds.bbci.co.uk/news/scotland/rss.xml?edition=uk" - }, /* - { - "count": 100, - "feed_url": "http://news.google.com/?output=rss" - },/* - { - "count": 100, - "feed_url": "http://feeds.feedburner.com/TheAtlantic" - }, /* - { - "count": 100, - "feed_url": "http://www.telegraph.co.uk/newsfeed/rss/news-uk_news.xml" - }, - { - "count": 100, - "feed_url": "http://www.telegraph.co.uk/news/worldnews/rss" - }, /* - { - "count": 100, - "feed_url": "http://www.guardian.co.uk/rssfeed/0,,1,00.xml" - }, - { - "count": 100, - "feed_url": "http://www.marketwatch.com/rss/topstories" - }, - { - "count": 100, - "feed_url": "http://feeds.feedburner.com/wingsoverscotland/FgKv" - }, - { - "count": 100, - "feed_url": "http://bellacaledonia.org.uk/feed/" - } - - */ - ] -} - - - -; var braider_options = { - feeds: feeds, - indent: " ", - date_sort_order: "desc", // Newest first - log_level: "warn", - dedupe_fields: ['link', 'guid'], - plugins_directories: [__dirname + "/plugins/"] + 'feeds': feeds, + 'indent': ' ', + 'date_sort_order': 'desc', // Newest first + 'log_level': 'warn', + 'dedupe_fields': ['link', 'guid'], + 'plugins_directories': [`${__dirname }/plugins/`] }; var rss_braider = RssBraider.createClient(braider_options); // Override logging level (debug, info, warn, err, off) -//rss_braider.logger.level('off'); +rss_braider.logger.level('error'); -rss_braider.processFeed('simple_test_feed', 'json', function(err, data) { - console.log('A'); +rss_braider.processFeed('simple_test_feed', 'json', function (err, data) { + if (err) + return console.log(err); + + var j = JSON.parse(data); + var ejsOutput = ejs.compile(str)(j); + + fs.writeFile(`${__dirname }/dist/${ outputFile }.html`, ejsOutput, function (err) { + if (err) + return console.log(err); + + console.log('The file was saved!'); + }); + + fs.writeFile(`${__dirname }/dist/${ outputFile }.json`, data, function (err) { if (err) { logger.error(err); - return err; + + return console.log(err); } - console.log('B'); - - var j = JSON.parse(data); - - console.log('C'); - var ejsOutput = ejs.compile(str)(j); - - console.log('D'); - fs.writeFile(__dirname + "/dist/" + outputFile + ".html", ejsOutput, function(err) { - - if (err) { - logger.error(err); - return console.error(err); - } - - console.log("The file was saved!"); - }); - - fs.writeFile(__dirname + "/dist/" + outputFile + ".json", data, function(err) { - if (err) { - logger.error(err); - return console.error(err); - } - - console.log("The file was saved!"); - }); + console.log('The file was saved!'); + }); +}); + +rss_braider.processFeed('simple_test_feed', 'rss', function (err, data) { + if (err) + return console.log(err); + + fs.writeFile(`${__dirname }/dist/${ outputFile }.xml`, data, function (err) { + if (err) + return console.log(err); + + console.log('The file was saved!'); + }); });