adding some extras to filtering

This commit is contained in:
Martin Donnelly 2016-04-26 10:06:09 +01:00
parent 168d08baa3
commit 9fcb85992b
2 changed files with 131 additions and 146 deletions

View File

@ -1,4 +1,4 @@
"use strict";
'use strict';
/**
* Created by Martin on 22/02/2016.
*/
@ -8,7 +8,7 @@ var http = require('http'), request = require('request'), cheerio = require(
var jsonfile = require('jsonfile'), fs = require('fs'), STRING = require(
'string');
var converter = require('html-to-markdown');
var zlib = require("zlib");
var zlib = require('zlib');
var log4js = require('log4js');
var logger = log4js.getLogger();
var URL = require('url');
@ -36,17 +36,17 @@ var generics = [
];
var specialHandlers = [{
url: 'www.reddit.com', fn: function (body, url) {
url: 'www.reddit.com', fn: function(body, url) {
return doReddit(body, url);
}
},
{
url: 'developer.android.com', fn: function (body, url) {
url: 'developer.android.com', fn: function(body, url) {
return doAndroidDeveloper(body, url);
}
},
{
url: 'www.engadget.com', fn: function (body, url) {
url: 'www.engadget.com', fn: function(body, url) {
return doEngadget(body, url);
}
}
@ -87,7 +87,9 @@ function cleaner(b) {
'#recipe-related-videos',
'#tnav',
'.footer',
'#tb-wrapper'
'#tb-wrapper',
'#comments',
'#menu'
];
@ -100,8 +102,8 @@ function cleaner(b) {
function insertBookmark(obj) {
logger.debug('Inserting into couch...');
// logger.info(util.inspect(obj));
dbCouch.insert(obj, function (err, body, header) {
// Logger.info(util.inspect(obj));
dbCouch.insert(obj, function(err, body, header) {
if (err) {
logger.error('Error inserting into couch');
return;
@ -116,51 +118,51 @@ function updateBookmark(obj, _id, _rev) {
_obj._id = _id;
_obj._rev = _rev;
dbCouch.insert(_obj, function (err, body, header) {
dbCouch.insert(_obj, function(err, body, header) {
if (err) {
logger.error('Error updating into couch');
return;
} else {
logger.info('I think we updated ok...');
busEmitter.emit("updateTagsDB");
busEmitter.emit('updateTagsDB');
}
});
logger.debug('Update done..');
}
var doInsertBookmark = (obj) => {
// logger.info('sendSocket: ' + JSON.stringify(obj));
// Logger.info('sendSocket: ' + JSON.stringify(obj));
insertBookmark(obj);
};
var doUpdateBookmark = (obj, _id, _rev) => {
// logger.info('sendSocket: ' + JSON.stringify(obj));
// Logger.info('sendSocket: ' + JSON.stringify(obj));
updateBookmark(obj, _id, _rev);
};
var doGetBookmark = (obj) => {
// logger.info('sendSocket: ' + JSON.stringify(obj));
// Logger.info('sendSocket: ' + JSON.stringify(obj));
genericGrab(obj);
};
var doGetBookmarkRedo = (obj) => {
// logger.info('sendSocket: ' + JSON.stringify(obj));
// Logger.info('sendSocket: ' + JSON.stringify(obj));
genericGrab(obj);
};
var doGetBookmarkRes = (url, res) => {
logger.debug('doGetBookmarkRes');
// logger.info('sendSocket: ' + JSON.stringify(obj));
// Logger.info('sendSocket: ' + JSON.stringify(obj));
genericGrab(url, res);
};
var doUpdateTagsDB = () => {
logger.debug('Update the tags database...');
dbCouch.view('getAllTags', 'getAllTags', function (err, body) {
dbCouch.view('getAllTags', 'getAllTags', function(err, body) {
var masterList = [];
if (!err) {
body.rows.forEach(function (doc) {
body.rows.forEach(function(doc) {
masterList = masterList.concat(doc.value);
});
@ -169,24 +171,22 @@ var doUpdateTagsDB = () => {
return self.indexOf(value) === index;
});
dbCouch.view('taglist', 'taglist', function (err, body) {
// logger.debug(body);
dbCouch.view('taglist', 'taglist', function(err, body) {
// Logger.debug(body);
if (!err) {
var outJSON = {};
body.rows.forEach(function (doc) {
body.rows.forEach(function(doc) {
doSaveTagsDB(doc.value, masterList);
});
}
else {
} else {
logger.error('NO TAG LIST EXISTS');
}
});
}
else {
} else {
}
});
@ -200,7 +200,7 @@ var doSaveTagsDB = (orig, newList) => {
_obj.taglist = newList;
dbCouch.insert(_obj, function (err, body, header) {
dbCouch.insert(_obj, function(err, body, header) {
if (err) {
logger.error('Error updating into couch');
return;
@ -221,8 +221,7 @@ busEmitter.on('getBookmarkRedo', doGetBookmarkRedo);
busEmitter.on('updateTagsDB', doUpdateTagsDB);
busEmitter.on('saveTagsDB', doSaveTagsDB);
function doEngadget(body, url)
{
function doEngadget(body, url) {
logger.info('GRABBING Engadget');
var obj = {}, tdihbody, i, urlObj, urlPrefix;
@ -239,8 +238,8 @@ function doEngadget(body, url)
urlPrefix = urlObj.protocol + '//' + urlObj.host + '/';
try {
tdihbody.find('IMG').each(function (i, elem) {
let s, src = $(this).attr("src");
tdihbody.find('IMG').each(function(i, elem) {
let s, src = $(this).attr('src');
if (src !== null) {
if (!STRING(src).startsWith('http')) {
@ -254,7 +253,7 @@ function doEngadget(body, url)
s = 'http://image.silvrtree.co.uk/900,fit/' + src;
$(this).attr("src", s);
$(this).attr('src', s);
}
});
@ -274,8 +273,7 @@ function doEngadget(body, url)
}
function doAndroidDeveloper(body, url)
{
function doAndroidDeveloper(body, url) {
logger.info('GRABBING AndroidDeveloper');
var obj = {}, tdihbody, i, urlObj, urlPrefix;
@ -292,8 +290,8 @@ function doAndroidDeveloper(body, url)
urlPrefix = urlObj.protocol + '//' + urlObj.host + '/';
try {
tdihbody.find('IMG').each(function (i, elem) {
let s, src = $(this).attr("src");
tdihbody.find('IMG').each(function(i, elem) {
let s, src = $(this).attr('src');
if (src !== null) {
if (!STRING(src).startsWith('http')) {
@ -307,7 +305,7 @@ function doAndroidDeveloper(body, url)
s = 'http://image.silvrtree.co.uk/900,fit/' + src;
$(this).attr("src", s);
$(this).attr('src', s);
}
});
@ -327,9 +325,8 @@ function doAndroidDeveloper(body, url)
}
function doReddit(body, url)
{
logger.info('GRABBING REDDIT');
function doReddit(body, url) {
logger.info('GRABBING REDDIT');
var obj = {}, tdihbody, i, urlObj, urlPrefix;
var $ = cheerio.load(body);
@ -337,13 +334,13 @@ logger.info('GRABBING REDDIT');
tdihbody = $('DIV.entry');
tdihbody.find('A.thumbnail').each(function (i, elem) {
tdihbody.find('A.thumbnail').each(function(i, elem) {
logger.warn($(this));
});
logger.info('++++++');
// logger.debug(tdihbody.html());
// Logger.debug(tdihbody.html());
logger.debug(tdihbody.length);
tdihbody = cleaner(tdihbody);
@ -381,15 +378,14 @@ function genericProcessor(body, url) {
tdihbody = cleaner(tdihbody);
logger.debug(title);
}
else {
} else {
logger.warn('Using whole body');
// bah. nothing to reduce so just grab the body, tidy it and use that
// Bah. nothing to reduce so just grab the body, tidy it and use that
tdihbody = $('BODY');
if (tdihbody.length === 0) {
tdihbody = $(":root");
tdihbody = $(':root');
}
@ -398,14 +394,14 @@ function genericProcessor(body, url) {
logger.debug(title);
}
// logger.info(util.inspect(tdihbody));
// Logger.info(util.inspect(tdihbody));
urlObj = URL.parse(url);
urlPrefix = urlObj.protocol + '//' + urlObj.host + '/';
try {
tdihbody.find('IMG').each(function (i, elem) {
let s, src = $(this).attr("src");
tdihbody.find('IMG').each(function(i, elem) {
let s, src = $(this).attr('src');
console.log('!!!!' + src);
if (src !== null && typeof src !== 'undefined') {
@ -420,7 +416,7 @@ function genericProcessor(body, url) {
s = 'http://image.silvrtree.co.uk/900,fit/' + src;
$(this).attr("src", s);
$(this).attr('src', s);
}
});
@ -446,40 +442,38 @@ function processBody(body, url, _id, _rev) {
// try to find a body to grab
// Try to find a body to grab
urlObj = URL.parse(url);
logger.debug('host:', urlObj.host);
var flag;
for (i=0;i<specialHandlers.length;i++)
{
for (i = 0;i < specialHandlers.length;i++) {
if (urlObj.host === specialHandlers[i].url) {
flag=true;
flag = true;
obj = specialHandlers[i].fn(body,url);
}
}
if (!flag) {
// do generic processing
// Do generic processing
obj = genericProcessor(body,url);
}
// logger.warn(obj.reduced);
// Logger.warn(obj.reduced);
obj.host = urlObj.host;
/* jsonfile.writeFile(jsonFile, obj, function (err) {
/* Jsonfile.writeFile(jsonFile, obj, function (err) {
console.error(err);
});*/
if (_id !== null) {
busEmitter.emit("updateBookmarkData", obj, _id, _rev);
}
else {
busEmitter.emit("saveBookmarkData", obj);
busEmitter.emit('updateBookmarkData', obj, _id, _rev);
} else {
busEmitter.emit('saveBookmarkData', obj);
}
return obj;
@ -492,8 +486,7 @@ function genericGrab(obj, res) {
if (typeof obj === 'string') {
logger.info(obj);
url = obj;
}
else {
} else {
url = obj.url;
_id = obj._id || null;
_ver = obj._rev || null;
@ -515,7 +508,7 @@ function genericGrab(obj, res) {
followAllRedirects: true
};
request(options, function (err, resp, body) {
request(options, function(err, resp, body) {
if (err)
throw err;
@ -528,26 +521,24 @@ function genericGrab(obj, res) {
var gunzip = zlib.createGunzip();
var jsonString = '';
resp.pipe(gunzip);
gunzip.on('data', function (chunk) {
gunzip.on('data', function(chunk) {
jsonString += chunk;
});
gunzip.on('end', function () {
// console.log((jsonString));
gunzip.on('end', function() {
// Console.log((jsonString));
callback(JSON.stringify(jsonString));
});
gunzip.on('error', function (e) {
gunzip.on('error', function(e) {
console.log(e);
});
}
else {
} else {
var b = processBody(body, url, _id, _ver);
if (res != null) {
res.render('grabbed');
}
}
}
else {
} else {
var b = processBody(body, url, _id, _ver);
if (res != null) {
res.render('grabbed', {data: b});
@ -557,22 +548,21 @@ function genericGrab(obj, res) {
});
}
router.get('/pocket', function (req, res) {
router.get('/pocket', function(req, res) {
logger.debug('list..');
dbCouch.view('pocketList', 'pocketList', function (err, body) {
dbCouch.view('pocketList', 'pocketList', function(err, body) {
if (!err) {
var outJSON = [];
body.rows.forEach(function (doc) {
body.rows.forEach(function(doc) {
var obj = {id: doc.id, entry: doc.value};
console.log(typeof obj.entry.tn);
console.log(typeof obj.entry.tn);
if (typeof obj.entry.tn === 'string') {
console.log('its a string:', typeof obj.entry.tn)
obj.entry.tn = 'http://image.silvrtree.co.uk/100,fit,q80/' + obj.entry.tn;
}
else {
} else {
obj.entry.tn = 'gfx/fm.png';
}
@ -584,9 +574,8 @@ console.log(typeof obj.entry.tn);
logger.debug(util.inspect(body));
logger.info(util.inspect(outJSON));
res.render('pocket', {data: outJSON});
}
else {
res.writeHead(500, {"ContentType": "application/json"});
} else {
res.writeHead(500, {ContentType: 'application/json'});
res.end(JSON.stringify({}));
}
});
@ -594,35 +583,34 @@ console.log(typeof obj.entry.tn);
router.get('/list', function (req, res) {
router.get('/list', function(req, res) {
logger.debug('list..');
dbCouch.view('titles', 'titles', function (err, body) {
dbCouch.view('titles', 'titles', function(err, body) {
if (!err) {
var outJSON = [];
body.rows.forEach(function (doc) {
body.rows.forEach(function(doc) {
outJSON.push({id: doc.id, title: doc.value});
});
//logger.debug(util.inspect(body));
res.writeHead(200, {"ContentType": "application/json"});
//Logger.debug(util.inspect(body));
res.writeHead(200, {ContentType: 'application/json'});
res.end(JSON.stringify({list: outJSON}));
}
else {
res.writeHead(500, {"ContentType": "application/json"});
} else {
res.writeHead(500, {ContentType: 'application/json'});
res.end(JSON.stringify({}));
}
});
});
router.get('/entry/:id', function (req, res) {
router.get('/entry/:id', function(req, res) {
logger.debug('entry..');
logger.debug(req.params.id);
dbCouch.get(req.params.id, function (err, body) {
dbCouch.get(req.params.id, function(err, body) {
if (!err) {
var outJSON = {};
@ -633,102 +621,98 @@ router.get('/entry/:id', function (req, res) {
outJSON.url = body.url;
outJSON.tags = body.tags || {solid: '', list: []};
//logger.debug(util.inspect(body));
res.writeHead(200, {"ContentType": "application/json"});
//Logger.debug(util.inspect(body));
res.writeHead(200, {ContentType: 'application/json'});
res.end(JSON.stringify(outJSON));
}
else {
res.writeHead(500, {"ContentType": "application/json"});
} else {
res.writeHead(500, {ContentType: 'application/json'});
res.end(JSON.stringify({}));
}
});
});
router.get('/tags', function (req, res) {
router.get('/tags', function(req, res) {
logger.debug('tag list..');
logger.debug(req.params.id);
dbCouch.view('taglist', 'taglist', function (err, body) {
dbCouch.view('taglist', 'taglist', function(err, body) {
if (!err) {
logger.debug(body);
var outJSON = [];
body.rows.forEach(function (doc) {
body.rows.forEach(function(doc) {
logger.info(doc.value.taglist);
if (doc.value[0] == req.params.id) {
outJSON = doc.value.taglist.sort();
}
});
//logger.debug(util.inspect(body));
res.writeHead(200, {"ContentType": "application/json"});
//Logger.debug(util.inspect(body));
res.writeHead(200, {ContentType: 'application/json'});
res.end(JSON.stringify({list: outJSON}));
}
else {
} else {
logger.error(err);
res.writeHead(500, {"ContentType": "application/json"});
res.writeHead(500, {ContentType: 'application/json'});
res.end(JSON.stringify({}));
}
});
});
router.get('/tags/:id', function (req, res) {
router.get('/tags/:id', function(req, res) {
logger.debug('entry..');
logger.debug(req.params.id);
dbCouch.view('getTagByKey', 'getTagByKey', function (err, body) {
dbCouch.view('getTagByKey', 'getTagByKey', function(err, body) {
if (!err) {
// logger.debug(body);
// Logger.debug(body);
var outJSON = [];
body.rows.forEach(function (doc) {
// logger.debug(doc);
body.rows.forEach(function(doc) {
// Logger.debug(doc);
if (doc.value[0] == req.params.id) {
outJSON.push({id: doc.id, title: doc.value[1]})
}
});
//logger.debug(util.inspect(body));
res.writeHead(200, {"ContentType": "application/json"});
//Logger.debug(util.inspect(body));
res.writeHead(200, {ContentType: 'application/json'});
res.end(JSON.stringify({list: outJSON}));
}
else {
} else {
logger.error(err);
res.writeHead(500, {"ContentType": "application/json"});
res.writeHead(500, {ContentType: 'application/json'});
res.end(JSON.stringify({}));
}
});
});
router.post('/add', function (req, res) {
router.post('/add', function(req, res) {
logger.debug('add entry..');
var t = req.body;
if (t.hasOwnProperty('url')) {
var url = JSON.parse(t.url.toString());
logger.debug(url);
busEmitter.emit("getBookmark", t);
}
else {
busEmitter.emit('getBookmark', t);
} else {
logger.error('No data block!');
}
res.writeHead(200, {"ContentType": "application/json"});
res.writeHead(200, {ContentType: 'application/json'});
res.end(JSON.stringify({adding: url}));
});
router.post('/tags', function (req, res) {
router.post('/tags', function(req, res) {
var t = req.body;
console.log(t);
logger.info('regetting:' + req.body._id);
dbCouch.get(req.body._id, function (err, body) {
dbCouch.get(req.body._id, function(err, body) {
if (!err) {
var obj = {};
@ -740,7 +724,7 @@ router.post('/tags', function (req, res) {
obj.tags = req.body.tags;
logger.info('Updating...');
busEmitter.emit("updateBookmarkData", obj, body._id, body._rev, res);
busEmitter.emit('updateBookmarkData', obj, body._id, body._rev, res);
var outJSON = {};
outJSON._id = body._id;
@ -750,20 +734,19 @@ router.post('/tags', function (req, res) {
outJSON.url = body.url;
outJSON.tags = req.body.tags;
//logger.debug(util.inspect(body));
res.writeHead(200, {"ContentType": "application/json"});
//Logger.debug(util.inspect(body));
res.writeHead(200, {ContentType: 'application/json'});
res.end(JSON.stringify(outJSON));
}
else {
res.writeHead(500, {"ContentType": "application/json"});
} else {
res.writeHead(500, {ContentType: 'application/json'});
res.end(JSON.stringify({}));
}
});
});
router.post('/redo', function (req, res) {
router.post('/redo', function(req, res) {
logger.debug('redoing entry..');
var t = req.body;
@ -771,30 +754,27 @@ router.post('/redo', function (req, res) {
if (t.hasOwnProperty('url')) {
var url = t.url.toString();
logger.debug(url);
busEmitter.emit("getBookmark", t);
}
else {
busEmitter.emit('getBookmark', t);
} else {
logger.error('No data block!');
}
res.writeHead(200, {"ContentType": "application/json"});
res.writeHead(200, {ContentType: 'application/json'});
res.end(JSON.stringify({adding: url}));
});
router.get('/new', function (req, res) {
router.get('/new', function(req, res) {
logger.debug('Save new');
busEmitter.emit("getBookmarkRes", req.query.url, res);
busEmitter.emit('getBookmarkRes', req.query.url, res);
});
router.post('/new', function (req, res) {
router.post('/new', function(req, res) {
logger.debug('Posted Save new');
logger.info( req.body);
if (Object.keys(req.body).length !== 0)
{
busEmitter.emit("getBookmarkRes", req.body.url, res);
}
else {
logger.info(req.body);
if (Object.keys(req.body).length !== 0) {
busEmitter.emit('getBookmarkRes', req.body.url, res);
} else {
res.status(422).end();
}

View File

@ -108,7 +108,12 @@
<%= data[i].entry.host %>
</div>
<div class="mui--text-body1 tags">
<%= data[i].entry.tags %>
<% if (typeof data[i].entry.tags !== 'undefined') { %>
TAGS:
<%=data[i].entry.tags.solid %>
<% } %>
</div>
</div>
</div>