martin/keeper
martin/keeper
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
Martin Donnelly 2016-03-21 10:41:03 +00:00
parent cf1efc6b17
commit 5e6d3d0a43
2 changed files with 169 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
.jshintrc
View File

@ -1,32 +1,41 @@
{ {
"node": true, "predef": [
"esnext": true, "Promise"
"bitwise": true, ],
"camelcase": true, "node": true,
"curly": true, "browser": false,
"eqeqeq": true, "boss": true,
"immed": true, "curly": true,
"indent": 2, "debug": false,
"latedef": true, "devel": true,
"newcap": true, "eqeqeq": true,
"noarg": true, "evil": true,
"quotmark": "single", "forin": false,
"regexp": true, "immed": false,
"undef": true, "laxbreak": false,
"unused": true, "newcap": true,
"strict": true, "noarg": true,
"trailing": true, "noempty": false,
"smarttabs": true, "nonew": false,
"white": true, "nomen": false,
"validthis": true, "onevar": false,
"browser" : true, "plusplus": false,
"jquery":true, "regexp": false,
"undef": true,
"globals": { "sub": true,
"strict": false,
"$": false, "white": false,
"EJS": false, "eqnull": true,
"MicroEvent": false "esnext": true,
"unused": true,
} "supernew": true,
"globals": {
"$": false,
"EJS": false,
"MicroEvent": false,
'express':false,
'http':false,
'request':false,
'cheerio':false
}
} }

245
server/keeper.js
View File

@ -3,8 +3,10 @@
* Created by Martin on 22/02/2016. * Created by Martin on 22/02/2016.
*/ */
var express = require('express'); var express = require('express');
var http = require('http'), request = require('request'), cheerio = require('cheerio'), util = require('util'); var http = require('http'), request = require('request'), cheerio = require(
var jsonfile = require('jsonfile'), fs = require('fs'), STRING = require('string'); 'cheerio'), util = require('util');
var jsonfile = require('jsonfile'), fs = require('fs'), STRING = require(
'string');
var zlib = require("zlib"); var zlib = require("zlib");
var log4js = require('log4js'); var log4js = require('log4js');
var logger = log4js.getLogger(); var logger = log4js.getLogger();
@ -20,43 +22,68 @@ var busEmitter = new EventEmitter();
var db_name = 'keeper'; var db_name = 'keeper';
var dbCouch = nano.use(db_name); var dbCouch = nano.use(db_name);
var bodyfile = __dirname + '/' + 'body.html'; var bodyfile = __dirname + '/' + 'body.html';
var htmlfile = __dirname + '/' + 'testoutput.html'; var htmlfile = __dirname + '/' + 'testoutput.html';
var generics = ['ARTICLE', 'div.content_column', 'div.post','div.page']; var generics = [
'ARTICLE',
'div.content_column',
'div.post',
'div.page',
'#recipe-single'
];
function cleaner(b) { function cleaner(b) {
var _b = b; var _b = b;
var unwanted = ['div#disqus_thread', 'SCRIPT', 'FOOTER', 'div.ssba', '.shareaholic-canvas', '.yarpp-related', 'div.dfad', 'div.postFooterShare', 'div#nextPrevLinks', '.post-comments','HEADER', '.post-title','#side-menu','.footer-container','#pre-footer','#cakephp-global-navigation', '.masthead','.breadcrumb-header']; var unwanted = [
'div#disqus_thread',
'SCRIPT',
'FOOTER',
'div.ssba',
'.shareaholic-canvas',
'.yarpp-related',
'div.dfad',
'div.postFooterShare',
'div#nextPrevLinks',
'.post-comments',
'HEADER',
'.post-title',
'#side-menu',
'.footer-container',
'#pre-footer',
'#cakephp-global-navigation',
'.masthead',
'.breadcrumb-header',
'.single-recipe-sidebar',
'#recipe-related-videos'
];
for (var i = 0; i < unwanted.length; i++) { for (var i = 0; i < unwanted.length; i++) {
_b.find(unwanted[i]).remove(); _b.find(unwanted[i]).remove();
} }
return _b; return _b;
} }
function insertBookmark(obj) { function insertBookmark(obj) {
logger.debug('Inserting into couch...'); logger.debug('Inserting into couch...');
logger.info(util.inspect(obj)); logger.info(util.inspect(obj));
dbCouch.insert(obj, function(err, body,header) { dbCouch.insert(obj, function(err, body, header) {
if (err) { if (err) {
logger.error('Error inserting into couch'); logger.error('Error inserting into couch');
return; return;
} }
}); });
logger.debug('Insert done..'); logger.debug('Insert done..');
} }
function updateBookmark(obj,_id, _rev) { function updateBookmark(obj, _id, _rev) {
logger.debug('Updating couch...'); logger.debug('Updating couch...');
var _obj = obj; var _obj = obj;
_obj._id = _id; _obj._id = _id;
_obj._rev = _rev; _obj._rev = _rev;
dbCouch.insert(_obj, function(err, body,header) { dbCouch.insert(_obj, function(err, body, header) {
if (err) { if (err) {
logger.error('Error updating into couch'); logger.error('Error updating into couch');
return; return;
@ -64,30 +91,30 @@ function updateBookmark(obj,_id, _rev) {
}); });
logger.debug('Update done..'); logger.debug('Update done..');
} }
var doInsertBookmark = (obj) =>{ var doInsertBookmark = (obj) => {
// logger.info('sendSocket: ' + JSON.stringify(obj));
insertBookmark(obj);
};
var doUpdateBookmark = (obj, _id, _rev) =>{
// logger.info('sendSocket: ' + JSON.stringify(obj)); // logger.info('sendSocket: ' + JSON.stringify(obj));
updateBookmark(obj,_id, _rev); insertBookmark(obj);
}; };
var doGetBookmark = (obj) =>{ var doUpdateBookmark = (obj, _id, _rev) => {
// logger.info('sendSocket: ' + JSON.stringify(obj)); // logger.info('sendSocket: ' + JSON.stringify(obj));
genericGrab(obj); updateBookmark(obj, _id, _rev);
}; };
var doGetBookmarkRedo = (obj) =>{ var doGetBookmark = (obj) => {
// logger.info('sendSocket: ' + JSON.stringify(obj)); // logger.info('sendSocket: ' + JSON.stringify(obj));
genericGrab(obj); genericGrab(obj);
}; };
var doGetBookmarkRes = (url,res) =>{ var doGetBookmarkRedo = (obj) => {
// logger.info('sendSocket: ' + JSON.stringify(obj));
genericGrab(obj);
};
var doGetBookmarkRes = (url, res) => {
logger.debug('doGetBookmarkRes'); logger.debug('doGetBookmarkRes');
// logger.info('sendSocket: ' + JSON.stringify(obj)); // logger.info('sendSocket: ' + JSON.stringify(obj));
genericGrab(url,res); genericGrab(url, res);
}; };
// Events // Events
@ -120,7 +147,8 @@ function processBody(body, url, _id, _rev) {
tdihbody = cleaner(tdihbody); tdihbody = cleaner(tdihbody);
logger.debug(title); logger.debug(title);
} else { }
else {
// bah. nothing to reduce so just grab the body, tidy it and use that // bah. nothing to reduce so just grab the body, tidy it and use that
tdihbody = $('BODY'); tdihbody = $('BODY');
logger.debug(tdihbody.length); logger.debug(tdihbody.length);
@ -131,7 +159,7 @@ function processBody(body, url, _id, _rev) {
urlObj = URL.parse(url); urlObj = URL.parse(url);
urlPrefix = urlObj.protocol + '//' + urlObj.host + '/'; urlPrefix = urlObj.protocol + '//' + urlObj.host + '/';
try { try {
tdihbody.find('IMG').each(function (i, elem) { tdihbody.find('IMG').each(function(i, elem) {
let s, src = $(this).attr("src"); let s, src = $(this).attr("src");
if (!STRING(src).startsWith('http')) { if (!STRING(src).startsWith('http')) {
@ -152,118 +180,107 @@ function processBody(body, url, _id, _rev) {
obj.reduced = STRING(tdihbody.html()).trim().s; obj.reduced = STRING(tdihbody.html()).trim().s;
obj.title = STRING(title).collapseWhitespace().s; obj.title = STRING(title).collapseWhitespace().s;
if (_id !== null ) if (_id !== null) {
{
busEmitter.emit("updateBookmarkData", obj, _id, _rev); busEmitter.emit("updateBookmarkData", obj, _id, _rev);
} }
else else {
{
busEmitter.emit("saveBookmarkData", obj); busEmitter.emit("saveBookmarkData", obj);
} }
return obj; return obj;
} }
function genericGrab(obj,res) { function genericGrab(obj, res) {
var url, _id=null, _ver=null; var url, _id = null, _ver = null;
if (typeof obj === 'string') if (typeof obj === 'string') {
{
logger.info(obj); logger.info(obj);
url =obj; url = obj;
} }
else else {
{ url = obj.url;
url = obj.url; _id = obj._id || null;
_id = obj._id || null; _ver = obj._rev || null;
_ver = obj._rev || null;
} }
logger.warn(typeof obj); logger.warn(typeof obj);
logger.info(url);
logger.info(url);
logger.info(_id); logger.info(_id);
logger.info(_ver); logger.info(_ver);
request(url, function (err, resp, body) { request(url, function(err, resp, body) {
if (err) if (err)
throw err; throw err;
if (resp.headers.hasOwnProperty('content-encoding')) { if (resp.headers.hasOwnProperty('content-encoding')) {
logger.warn('content-encoding'); logger.warn('content-encoding');
if (resp.headers['content-encoding'] == 'gzip') { if (resp.headers['content-encoding'] == 'gzip') {
// to test http://chaosinthekitchen.com/2009/07/lime-and-coconut-chicken/ // to test http://chaosinthekitchen.com/2009/07/lime-and-coconut-chicken/
var gunzip = zlib.createGunzip();
var gunzip = zlib.createGunzip(); var jsonString = '';
var jsonString = ''; resp.pipe(gunzip);
resp.pipe(gunzip); gunzip.on('data', function(chunk) {
gunzip.on('data', function (chunk) { jsonString += chunk;
jsonString += chunk; });
}); gunzip.on('end', function() {
gunzip.on('end', function () { console.log((jsonString));
console.log((jsonString)); callback(JSON.stringify(jsonString));
callback(JSON.stringify(jsonString)); });
}); gunzip.on('error', function(e) {
gunzip.on('error', function (e) { console.log(e);
console.log(e); });
}); }
} else {
else var b = processBody(body, url, _id, _ver);
{ if (res != null) {
var b = processBody(body,url, _id, _ver); res.render('grabbed');
if (res != null)
{
res.render('grabbed');
}
}
} else
{
var b = processBody(body,url, _id, _ver);
if (res != null)
{
res.render('grabbed',{data:b});
}
} }
}
}); }
else {
var b = processBody(body, url, _id, _ver);
if (res != null) {
res.render('grabbed', {data: b});
}
}
});
} }
router.get('/list', function (req, res) { router.get('/list', function(req, res) {
logger.debug('list..'); logger.debug('list..');
dbCouch.view('titles','titles',function(err, body) { dbCouch.view('titles', 'titles', function(err, body) {
if (!err) { if (!err) {
var outJSON = []; var outJSON = [];
body.rows.forEach(function(doc) { body.rows.forEach(function(doc) {
outJSON.push({id:doc.id, title:doc.value }) outJSON.push({id: doc.id, title: doc.value})
}); });
//logger.debug(util.inspect(body)); //logger.debug(util.inspect(body));
res.writeHead(200, {"ContentType": "application/json"}); res.writeHead(200, {"ContentType": "application/json"});
res.end(JSON.stringify({list: outJSON})); res.end(JSON.stringify({list: outJSON}));
} else }
{ else {
res.writeHead(500, {"ContentType": "application/json"}); res.writeHead(500, {"ContentType": "application/json"});
res.end(JSON.stringify({})); res.end(JSON.stringify({}));
} }
}); });
}); });
router.get('/entry/:id', function (req, res) { router.get('/entry/:id', function(req, res) {
logger.debug('entry..'); logger.debug('entry..');
logger.debug(req.params.id); logger.debug(req.params.id);
dbCouch.get(req.params.id,function(err, body) { dbCouch.get(req.params.id, function(err, body) {
if (!err) { if (!err) {
var outJSON = {}; var outJSON = {};
@ -276,8 +293,8 @@ router.get('/entry/:id', function (req, res) {
res.writeHead(200, {"ContentType": "application/json"}); res.writeHead(200, {"ContentType": "application/json"});
res.end(JSON.stringify(outJSON)); res.end(JSON.stringify(outJSON));
} else }
{ else {
res.writeHead(500, {"ContentType": "application/json"}); res.writeHead(500, {"ContentType": "application/json"});
res.end(JSON.stringify({})); res.end(JSON.stringify({}));
} }
@ -285,24 +302,24 @@ router.get('/entry/:id', function (req, res) {
}); });
router.post('/add', function (req, res) { router.post('/add', function(req, res) {
logger.debug('add entry..'); logger.debug('add entry..');
var t = req.body; var t = req.body;
if (t.hasOwnProperty('url')) { if (t.hasOwnProperty('url')) {
var url = JSON.parse(t.url.toString()); var url = JSON.parse(t.url.toString());
logger.debug(url); logger.debug(url);
busEmitter.emit("getBookmark", t); busEmitter.emit("getBookmark", t);
} }
else { else {
logger.error('No data block!'); logger.error('No data block!');
} }
res.writeHead(200, {"ContentType": "application/json"}); res.writeHead(200, {"ContentType": "application/json"});
res.end(JSON.stringify({adding: url})); res.end(JSON.stringify({adding: url}));
}); });
router.post('/redo', function (req, res) { router.post('/redo', function(req, res) {
logger.debug('redoing entry..'); logger.debug('redoing entry..');
var t = req.body; var t = req.body;
@ -320,11 +337,9 @@ router.post('/redo', function (req, res) {
}); });
router.get('/new', function(req, res) {
router.get('/new', function (req, res) {
logger.debug('Save new'); logger.debug('Save new');
busEmitter.emit("getBookmarkRes", req.query.url ,res); busEmitter.emit("getBookmarkRes", req.query.url, res);
}); });