keeper/server/keeper.js
2016-04-13 14:37:14 +01:00

806 lines
18 KiB
JavaScript

"use strict";
/**
* Created by Martin on 22/02/2016.
*/
var express = require('express');
var http = require('http'), request = require('request'), cheerio = require(
'cheerio'), util = require('util');
var jsonfile = require('jsonfile'), fs = require('fs'), STRING = require(
'string');
var converter = require('html-to-markdown');
var zlib = require("zlib");
var log4js = require('log4js');
var logger = log4js.getLogger();
var URL = require('url');
var router = express.Router();
var EventEmitter = require('events');
var nano = require('nano')('http://localhost:5984');
var busEmitter = new EventEmitter();
var db_name = 'keeper';
var dbCouch = nano.use(db_name);
var jsonFile = __dirname + '/' + 'output.json';
var bodyfile = __dirname + '/' + 'body.html';
var htmlfile = __dirname + '/' + 'testoutput.html';
var generics = [
'ARTICLE',
'div.content_column',
'div.post',
'div.page',
'#recipe-single',
'div.content.body'
];
var specialHandlers = [{
url: 'www.reddit.com', fn: function (body, url) {
return doReddit(body, url);
}
},
{
url: 'developer.android.com', fn: function (body, url) {
return doAndroidDeveloper(body, url);
}
},
{
url: 'www.engadget.com', fn: function (body, url) {
return doEngadget(body, url);
}
}
];
function cleaner(b) {
var _b = b;
var unwanted = [
'LINK',
'META',
'TITLE',
'div#disqus_thread',
'SCRIPT',
'FOOTER',
'div.ssba',
'.shareaholic-canvas',
'.yarpp-related',
'div.dfad',
'div.postFooterShare',
'div#nextPrevLinks',
'.post-comments',
'HEADER',
'.post-title',
'#side-menu',
'.footer-container',
'#pre-footer',
'#cakephp-global-navigation',
'.masthead',
'.breadcrumb-header',
'.single-recipe-sidebar',
'#recipe-related-videos',
'#tnav',
'.footer',
'#tb-wrapper'
];
for (var i = 0; i < unwanted.length; i++) {
_b.find(unwanted[i]).remove();
}
return _b;
}
function insertBookmark(obj) {
logger.debug('Inserting into couch...');
// logger.info(util.inspect(obj));
dbCouch.insert(obj, function (err, body, header) {
if (err) {
logger.error('Error inserting into couch');
return;
}
});
logger.debug('Insert done..');
}
function updateBookmark(obj, _id, _rev) {
logger.debug('Updating couch...');
var _obj = obj;
_obj._id = _id;
_obj._rev = _rev;
dbCouch.insert(_obj, function (err, body, header) {
if (err) {
logger.error('Error updating into couch');
return;
} else {
logger.info('I think we updated ok...');
busEmitter.emit("updateTagsDB");
}
});
logger.debug('Update done..');
}
var doInsertBookmark = (obj) => {
// logger.info('sendSocket: ' + JSON.stringify(obj));
insertBookmark(obj);
};
var doUpdateBookmark = (obj, _id, _rev) => {
// logger.info('sendSocket: ' + JSON.stringify(obj));
updateBookmark(obj, _id, _rev);
};
var doGetBookmark = (obj) => {
// logger.info('sendSocket: ' + JSON.stringify(obj));
genericGrab(obj);
};
var doGetBookmarkRedo = (obj) => {
// logger.info('sendSocket: ' + JSON.stringify(obj));
genericGrab(obj);
};
var doGetBookmarkRes = (url, res) => {
logger.debug('doGetBookmarkRes');
// logger.info('sendSocket: ' + JSON.stringify(obj));
genericGrab(url, res);
};
var doUpdateTagsDB = () => {
logger.debug('Update the tags database...');
dbCouch.view('getAllTags', 'getAllTags', function (err, body) {
var masterList = [];
if (!err) {
body.rows.forEach(function (doc) {
masterList = masterList.concat(doc.value);
});
masterList = masterList.filter((value, index, self) => {
return self.indexOf(value) === index;
});
dbCouch.view('taglist', 'taglist', function (err, body) {
// logger.debug(body);
if (!err) {
var outJSON = {};
body.rows.forEach(function (doc) {
doSaveTagsDB(doc.value, masterList);
});
}
else {
logger.error('NO TAG LIST EXISTS');
}
});
}
else {
}
});
};
var doSaveTagsDB = (orig, newList) => {
logger.debug('doSaveTagsDB');
var _obj = orig;
_obj.taglist = newList;
dbCouch.insert(_obj, function (err, body, header) {
if (err) {
logger.error('Error updating into couch');
return;
} else {
logger.info('Updated the tags list...');
}
});
};
// Events
busEmitter.on('saveBookmarkData', doInsertBookmark);
busEmitter.on('updateBookmarkData', doUpdateBookmark);
busEmitter.on('getBookmark', doGetBookmark);
busEmitter.on('getBookmarkRes', doGetBookmarkRes);
busEmitter.on('getBookmarkRedo', doGetBookmarkRedo);
busEmitter.on('updateTagsDB', doUpdateTagsDB);
busEmitter.on('saveTagsDB', doSaveTagsDB);
function doEngadget(body, url)
{
logger.info('GRABBING Engadget');
var obj = {}, tdihbody, i, urlObj, urlPrefix;
var $ = cheerio.load(body);
var title = $('TITLE').text();
tdihbody = $('DIV#page_body');
logger.debug('Length:' , tdihbody.length);
tdihbody = cleaner(tdihbody);
logger.debug('Title: ', title);
urlObj = URL.parse(url);
urlPrefix = urlObj.protocol + '//' + urlObj.host + '/';
try {
tdihbody.find('IMG').each(function (i, elem) {
let s, src = $(this).attr("src");
if (src !== null) {
if (!STRING(src).startsWith('http')) {
logger.debug('Stripping:' + src);
src = urlPrefix + STRING(src).stripLeft('/').trim().s;
}
if (typeof obj.thumbnail === 'undefined') {
obj.thumbnail = src;
}
s = 'http://image.silvrtree.co.uk/900,fit/' + src;
$(this).attr("src", s);
}
});
}
catch (e) {
logger.error(e);
}
obj.url = STRING(url).trim().s;
obj.html = $.html();
obj.reduced = STRING(tdihbody.html()).trim().s;
obj.nib = STRING(tdihbody.text()).collapseWhitespace().trim().left(300).s;
obj.title = STRING(title).collapseWhitespace().s;
obj.markdown = converter.convert(obj.reduced);
return obj;
}
function doAndroidDeveloper(body, url)
{
logger.info('GRABBING AndroidDeveloper');
var obj = {}, tdihbody, i, urlObj, urlPrefix;
var $ = cheerio.load(body);
var title = $('TITLE').text();
tdihbody = $('DIV.jd-descr');
logger.debug(tdihbody.length);
tdihbody = cleaner(tdihbody);
logger.debug(title);
urlObj = URL.parse(url);
urlPrefix = urlObj.protocol + '//' + urlObj.host + '/';
try {
tdihbody.find('IMG').each(function (i, elem) {
let s, src = $(this).attr("src");
if (src !== null) {
if (!STRING(src).startsWith('http')) {
logger.debug('Stripping:' + src);
src = urlPrefix + STRING(src).stripLeft('/').trim().s;
}
if (typeof obj.thumbnail === 'undefined') {
obj.thumbnail = src;
}
s = 'http://image.silvrtree.co.uk/900,fit/' + src;
$(this).attr("src", s);
}
});
}
catch (e) {
logger.error(e);
}
obj.url = STRING(url).trim().s;
obj.html = $.html();
obj.reduced = STRING(tdihbody.html()).trim().s;
obj.nib = STRING(tdihbody.text()).collapseWhitespace().trim().left(300).s;
obj.title = STRING(title).collapseWhitespace().s;
obj.markdown = converter.convert(obj.reduced);
return obj;
}
function doReddit(body, url)
{
logger.info('GRABBING REDDIT');
var obj = {}, tdihbody, i, urlObj, urlPrefix;
var $ = cheerio.load(body);
var title = $('TITLE').text();
tdihbody = $('DIV.entry');
tdihbody.find('A.thumbnail').each(function (i, elem) {
logger.warn($(this));
});
logger.info('++++++');
// logger.debug(tdihbody.html());
logger.debug(tdihbody.length);
tdihbody = cleaner(tdihbody);
logger.debug(title);
obj.url = STRING(url).trim().s;
obj.html = $.html();
obj.reduced = STRING(tdihbody.html()).trim().s;
obj.nib = STRING(tdihbody.text()).collapseWhitespace().trim().left(300).s;
obj.title = STRING(title).collapseWhitespace().s;
obj.markdown = converter.convert(obj.reduced);
return obj;
}
function genericProcessor(body, url) {
logger.info('USING DEFAULT PROCESSOR');
var obj = {}, tdihbody, i, urlObj, urlPrefix;
var $ = cheerio.load(body);
var title = $('TITLE').text();
i = 0;
while (($(generics[i]).length == 0) && (i < generics.length)) {
i++;
}
logger.debug(i);
if (i < generics.length) {
logger.warn('Used a generic');
tdihbody = $(generics[i]);
logger.debug(tdihbody.length);
tdihbody = cleaner(tdihbody);
logger.debug(title);
}
else {
logger.warn('Using whole body');
// bah. nothing to reduce so just grab the body, tidy it and use that
tdihbody = $('BODY');
if (tdihbody.length === 0) {
tdihbody = $(":root");
}
logger.debug(tdihbody.length);
tdihbody = cleaner(tdihbody);
logger.debug(title);
}
// logger.info(util.inspect(tdihbody));
urlObj = URL.parse(url);
urlPrefix = urlObj.protocol + '//' + urlObj.host + '/';
try {
tdihbody.find('IMG').each(function (i, elem) {
let s, src = $(this).attr("src");
console.log('!!!!' + src);
if (src !== null && typeof src !== 'undefined') {
if (!STRING(src).startsWith('http')) {
logger.debug('Stripping:' + src);
src = urlPrefix + STRING(src).stripLeft('/').trim().s;
}
if (typeof obj.thumbnail === 'undefined') {
obj.thumbnail = src;
}
s = 'http://image.silvrtree.co.uk/900,fit/' + src;
$(this).attr("src", s);
}
});
}
catch (e) {
logger.error(e);
}
obj.url = STRING(url).trim().s;
obj.html = $.html();
obj.reduced = STRING(tdihbody.html()).trim().s;
obj.nib = STRING(tdihbody.text()).collapseWhitespace().trim().left(300).s;
obj.title = STRING(title).collapseWhitespace().s;
obj.markdown = converter.convert(obj.reduced);
return obj;
}
function processBody(body, url, _id, _rev) {
var obj = {}, i, urlObj, urlPrefix;
// try to find a body to grab
urlObj = URL.parse(url);
logger.debug('host:', urlObj.host);
var flag;
for (i=0;i<specialHandlers.length;i++)
{
if (urlObj.host === specialHandlers[i].url) {
flag=true;
obj = specialHandlers[i].fn(body,url);
}
}
if (!flag) {
// do generic processing
obj = genericProcessor(body,url);
}
// logger.warn(obj.reduced);
obj.host = urlObj.host;
/* jsonfile.writeFile(jsonFile, obj, function (err) {
console.error(err);
});*/
if (_id !== null) {
busEmitter.emit("updateBookmarkData", obj, _id, _rev);
}
else {
busEmitter.emit("saveBookmarkData", obj);
}
return obj;
}
function genericGrab(obj, res) {
var url, _id = null, _ver = null;
if (typeof obj === 'string') {
logger.info(obj);
url = obj;
}
else {
url = obj.url;
_id = obj._id || null;
_ver = obj._rev || null;
}
logger.warn(typeof obj);
logger.info(url);
logger.info(_id);
logger.info(_ver);
var options = {
url: url,
headers: {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36'
},
jar: true,
followRedirect: true,
followAllRedirects: true
};
request(options, function (err, resp, body) {
if (err)
throw err;
if (resp.headers.hasOwnProperty('content-encoding')) {
logger.warn('content-encoding');
if (resp.headers['content-encoding'] == 'gzip') {
// to test http://chaosinthekitchen.com/2009/07/lime-and-coconut-chicken/
var gunzip = zlib.createGunzip();
var jsonString = '';
resp.pipe(gunzip);
gunzip.on('data', function (chunk) {
jsonString += chunk;
});
gunzip.on('end', function () {
// console.log((jsonString));
callback(JSON.stringify(jsonString));
});
gunzip.on('error', function (e) {
console.log(e);
});
}
else {
var b = processBody(body, url, _id, _ver);
if (res != null) {
res.render('grabbed');
}
}
}
else {
var b = processBody(body, url, _id, _ver);
if (res != null) {
res.render('grabbed', {data: b});
}
}
});
}
router.get('/pocket', function (req, res) {
logger.debug('list..');
dbCouch.view('pocketList', 'pocketList', function (err, body) {
if (!err) {
var outJSON = [];
body.rows.forEach(function (doc) {
var obj = {id: doc.id, entry: doc.value};
console.log(typeof obj.entry.tn);
if (typeof obj.entry.tn === 'string') {
console.log('its a string:', typeof obj.entry.tn)
obj.entry.tn = 'http://image.silvrtree.co.uk/100,fit,q80/' + obj.entry.tn;
}
else {
obj.entry.tn = 'gfx/fm.png';
}
outJSON.push(obj);
});
logger.debug(util.inspect(body));
logger.info(util.inspect(outJSON));
res.render('pocket', {data: outJSON});
}
else {
res.writeHead(500, {"ContentType": "application/json"});
res.end(JSON.stringify({}));
}
});
});
router.get('/list', function (req, res) {
logger.debug('list..');
dbCouch.view('titles', 'titles', function (err, body) {
if (!err) {
var outJSON = [];
body.rows.forEach(function (doc) {
outJSON.push({id: doc.id, title: doc.value});
});
//logger.debug(util.inspect(body));
res.writeHead(200, {"ContentType": "application/json"});
res.end(JSON.stringify({list: outJSON}));
}
else {
res.writeHead(500, {"ContentType": "application/json"});
res.end(JSON.stringify({}));
}
});
});
router.get('/entry/:id', function (req, res) {
logger.debug('entry..');
logger.debug(req.params.id);
dbCouch.get(req.params.id, function (err, body) {
if (!err) {
var outJSON = {};
outJSON._id = body._id;
outJSON._rev = body._rev;
outJSON.title = body.title;
outJSON.reduced = body.reduced;
outJSON.url = body.url;
outJSON.tags = body.tags || {solid: '', list: []};
//logger.debug(util.inspect(body));
res.writeHead(200, {"ContentType": "application/json"});
res.end(JSON.stringify(outJSON));
}
else {
res.writeHead(500, {"ContentType": "application/json"});
res.end(JSON.stringify({}));
}
});
});
router.get('/tags', function (req, res) {
logger.debug('tag list..');
logger.debug(req.params.id);
dbCouch.view('taglist', 'taglist', function (err, body) {
if (!err) {
logger.debug(body);
var outJSON = [];
body.rows.forEach(function (doc) {
logger.info(doc.value.taglist);
if (doc.value[0] == req.params.id) {
outJSON = doc.value.taglist.sort();
}
});
//logger.debug(util.inspect(body));
res.writeHead(200, {"ContentType": "application/json"});
res.end(JSON.stringify({list: outJSON}));
}
else {
logger.error(err);
res.writeHead(500, {"ContentType": "application/json"});
res.end(JSON.stringify({}));
}
});
});
router.get('/tags/:id', function (req, res) {
logger.debug('entry..');
logger.debug(req.params.id);
dbCouch.view('getTagByKey', 'getTagByKey', function (err, body) {
if (!err) {
// logger.debug(body);
var outJSON = [];
body.rows.forEach(function (doc) {
// logger.debug(doc);
if (doc.value[0] == req.params.id) {
outJSON.push({id: doc.id, title: doc.value[1]})
}
});
//logger.debug(util.inspect(body));
res.writeHead(200, {"ContentType": "application/json"});
res.end(JSON.stringify({list: outJSON}));
}
else {
logger.error(err);
res.writeHead(500, {"ContentType": "application/json"});
res.end(JSON.stringify({}));
}
});
});
router.post('/add', function (req, res) {
logger.debug('add entry..');
var t = req.body;
if (t.hasOwnProperty('url')) {
var url = JSON.parse(t.url.toString());
logger.debug(url);
busEmitter.emit("getBookmark", t);
}
else {
logger.error('No data block!');
}
res.writeHead(200, {"ContentType": "application/json"});
res.end(JSON.stringify({adding: url}));
});
router.post('/tags', function (req, res) {
var t = req.body;
console.log(t);
logger.info('regetting:' + req.body._id);
dbCouch.get(req.body._id, function (err, body) {
if (!err) {
var obj = {};
obj.url = body.url;
obj.html = body.html;
obj.reduced = body.reduced;
obj.title = body.title;
obj.tags = req.body.tags;
logger.info('Updating...');
busEmitter.emit("updateBookmarkData", obj, body._id, body._rev, res);
var outJSON = {};
outJSON._id = body._id;
outJSON._rev = body._rev;
outJSON.title = body.title;
outJSON.reduced = body.reduced;
outJSON.url = body.url;
outJSON.tags = req.body.tags;
//logger.debug(util.inspect(body));
res.writeHead(200, {"ContentType": "application/json"});
res.end(JSON.stringify(outJSON));
}
else {
res.writeHead(500, {"ContentType": "application/json"});
res.end(JSON.stringify({}));
}
});
});
router.post('/redo', function (req, res) {
logger.debug('redoing entry..');
var t = req.body;
console.log(t);
if (t.hasOwnProperty('url')) {
var url = t.url.toString();
logger.debug(url);
busEmitter.emit("getBookmark", t);
}
else {
logger.error('No data block!');
}
res.writeHead(200, {"ContentType": "application/json"});
res.end(JSON.stringify({adding: url}));
});
router.get('/new', function (req, res) {
logger.debug('Save new');
busEmitter.emit("getBookmarkRes", req.query.url, res);
});
router.post('/new', function (req, res) {
logger.debug('Posted Save new');
logger.info( req.body);
if (Object.keys(req.body).length !== 0)
{
busEmitter.emit("getBookmarkRes", req.body.url, res);
}
else {
res.status(422).end();
}
});
busEmitter.emit('updateTagsDB');
module.exports = router;