From 5e6d3d0a437af7e1dff99fa7d88f8322bc510d45 Mon Sep 17 00:00:00 2001 From: Martin Donnelly Date: Mon, 21 Mar 2016 10:41:03 +0000 Subject: [PATCH] update --- .jshintrc | 69 +++++++------ server/keeper.js | 245 +++++++++++++++++++++++++---------------------- 2 files changed, 169 insertions(+), 145 deletions(-) diff --git a/.jshintrc b/.jshintrc index 153e75f..bfce29a 100644 --- a/.jshintrc +++ b/.jshintrc @@ -1,32 +1,41 @@ { - "node": true, - "esnext": true, - "bitwise": true, - "camelcase": true, - "curly": true, - "eqeqeq": true, - "immed": true, - "indent": 2, - "latedef": true, - "newcap": true, - "noarg": true, - "quotmark": "single", - "regexp": true, - "undef": true, - "unused": true, - "strict": true, - "trailing": true, - "smarttabs": true, - "white": true, - "validthis": true, - "browser" : true, - "jquery":true, - - "globals": { - - "$": false, - "EJS": false, - "MicroEvent": false - - } + "predef": [ + "Promise" + ], + "node": true, + "browser": false, + "boss": true, + "curly": true, + "debug": false, + "devel": true, + "eqeqeq": true, + "evil": true, + "forin": false, + "immed": false, + "laxbreak": false, + "newcap": true, + "noarg": true, + "noempty": false, + "nonew": false, + "nomen": false, + "onevar": false, + "plusplus": false, + "regexp": false, + "undef": true, + "sub": true, + "strict": false, + "white": false, + "eqnull": true, + "esnext": true, + "unused": true, + "supernew": true, + "globals": { + "$": false, + "EJS": false, + "MicroEvent": false, + 'express':false, + 'http':false, + 'request':false, + 'cheerio':false + } } diff --git a/server/keeper.js b/server/keeper.js index 2e17f97..161768d 100644 --- a/server/keeper.js +++ b/server/keeper.js @@ -3,8 +3,10 @@ * Created by Martin on 22/02/2016. */ var express = require('express'); -var http = require('http'), request = require('request'), cheerio = require('cheerio'), util = require('util'); -var jsonfile = require('jsonfile'), fs = require('fs'), STRING = require('string'); +var http = require('http'), request = require('request'), cheerio = require( + 'cheerio'), util = require('util'); +var jsonfile = require('jsonfile'), fs = require('fs'), STRING = require( + 'string'); var zlib = require("zlib"); var log4js = require('log4js'); var logger = log4js.getLogger(); @@ -20,43 +22,68 @@ var busEmitter = new EventEmitter(); var db_name = 'keeper'; var dbCouch = nano.use(db_name); - var bodyfile = __dirname + '/' + 'body.html'; var htmlfile = __dirname + '/' + 'testoutput.html'; -var generics = ['ARTICLE', 'div.content_column', 'div.post','div.page']; - +var generics = [ + 'ARTICLE', + 'div.content_column', + 'div.post', + 'div.page', + '#recipe-single' +]; function cleaner(b) { - var _b = b; + var _b = b; - var unwanted = ['div#disqus_thread', 'SCRIPT', 'FOOTER', 'div.ssba', '.shareaholic-canvas', '.yarpp-related', 'div.dfad', 'div.postFooterShare', 'div#nextPrevLinks', '.post-comments','HEADER', '.post-title','#side-menu','.footer-container','#pre-footer','#cakephp-global-navigation', '.masthead','.breadcrumb-header']; + var unwanted = [ + 'div#disqus_thread', + 'SCRIPT', + 'FOOTER', + 'div.ssba', + '.shareaholic-canvas', + '.yarpp-related', + 'div.dfad', + 'div.postFooterShare', + 'div#nextPrevLinks', + '.post-comments', + 'HEADER', + '.post-title', + '#side-menu', + '.footer-container', + '#pre-footer', + '#cakephp-global-navigation', + '.masthead', + '.breadcrumb-header', + '.single-recipe-sidebar', + '#recipe-related-videos' + ]; - for (var i = 0; i < unwanted.length; i++) { - _b.find(unwanted[i]).remove(); - } + for (var i = 0; i < unwanted.length; i++) { + _b.find(unwanted[i]).remove(); + } - return _b; + return _b; } function insertBookmark(obj) { logger.debug('Inserting into couch...'); logger.info(util.inspect(obj)); - dbCouch.insert(obj, function(err, body,header) { + dbCouch.insert(obj, function(err, body, header) { if (err) { logger.error('Error inserting into couch'); return; } }); - logger.debug('Insert done..'); + logger.debug('Insert done..'); } -function updateBookmark(obj,_id, _rev) { +function updateBookmark(obj, _id, _rev) { logger.debug('Updating couch...'); var _obj = obj; _obj._id = _id; _obj._rev = _rev; - dbCouch.insert(_obj, function(err, body,header) { + dbCouch.insert(_obj, function(err, body, header) { if (err) { logger.error('Error updating into couch'); return; @@ -64,30 +91,30 @@ function updateBookmark(obj,_id, _rev) { }); logger.debug('Update done..'); } -var doInsertBookmark = (obj) =>{ - // logger.info('sendSocket: ' + JSON.stringify(obj)); - insertBookmark(obj); -}; - -var doUpdateBookmark = (obj, _id, _rev) =>{ +var doInsertBookmark = (obj) => { // logger.info('sendSocket: ' + JSON.stringify(obj)); - updateBookmark(obj,_id, _rev); + insertBookmark(obj); }; -var doGetBookmark = (obj) =>{ - // logger.info('sendSocket: ' + JSON.stringify(obj)); - genericGrab(obj); +var doUpdateBookmark = (obj, _id, _rev) => { + // logger.info('sendSocket: ' + JSON.stringify(obj)); + updateBookmark(obj, _id, _rev); }; -var doGetBookmarkRedo = (obj) =>{ +var doGetBookmark = (obj) => { // logger.info('sendSocket: ' + JSON.stringify(obj)); genericGrab(obj); }; -var doGetBookmarkRes = (url,res) =>{ +var doGetBookmarkRedo = (obj) => { + // logger.info('sendSocket: ' + JSON.stringify(obj)); + genericGrab(obj); +}; + +var doGetBookmarkRes = (url, res) => { logger.debug('doGetBookmarkRes'); // logger.info('sendSocket: ' + JSON.stringify(obj)); - genericGrab(url,res); + genericGrab(url, res); }; // Events @@ -120,7 +147,8 @@ function processBody(body, url, _id, _rev) { tdihbody = cleaner(tdihbody); logger.debug(title); - } else { + } + else { // bah. nothing to reduce so just grab the body, tidy it and use that tdihbody = $('BODY'); logger.debug(tdihbody.length); @@ -131,7 +159,7 @@ function processBody(body, url, _id, _rev) { urlObj = URL.parse(url); urlPrefix = urlObj.protocol + '//' + urlObj.host + '/'; try { - tdihbody.find('IMG').each(function (i, elem) { + tdihbody.find('IMG').each(function(i, elem) { let s, src = $(this).attr("src"); if (!STRING(src).startsWith('http')) { @@ -152,118 +180,107 @@ function processBody(body, url, _id, _rev) { obj.reduced = STRING(tdihbody.html()).trim().s; obj.title = STRING(title).collapseWhitespace().s; - if (_id !== null ) - { + if (_id !== null) { busEmitter.emit("updateBookmarkData", obj, _id, _rev); } - else - { + else { busEmitter.emit("saveBookmarkData", obj); } - return obj; } -function genericGrab(obj,res) { +function genericGrab(obj, res) { -var url, _id=null, _ver=null; + var url, _id = null, _ver = null; - if (typeof obj === 'string') - { + if (typeof obj === 'string') { logger.info(obj); - url =obj; + url = obj; } - else - { - url = obj.url; - _id = obj._id || null; - _ver = obj._rev || null; + else { + url = obj.url; + _id = obj._id || null; + _ver = obj._rev || null; } - logger.warn(typeof obj); - - logger.info(url); + logger.info(url); logger.info(_id); logger.info(_ver); - request(url, function (err, resp, body) { - if (err) - throw err; + request(url, function(err, resp, body) { + if (err) + throw err; - if (resp.headers.hasOwnProperty('content-encoding')) { - logger.warn('content-encoding'); - if (resp.headers['content-encoding'] == 'gzip') { + if (resp.headers.hasOwnProperty('content-encoding')) { + logger.warn('content-encoding'); + if (resp.headers['content-encoding'] == 'gzip') { - // to test http://chaosinthekitchen.com/2009/07/lime-and-coconut-chicken/ + // to test http://chaosinthekitchen.com/2009/07/lime-and-coconut-chicken/ - - var gunzip = zlib.createGunzip(); - var jsonString = ''; - resp.pipe(gunzip); - gunzip.on('data', function (chunk) { - jsonString += chunk; - }); - gunzip.on('end', function () { - console.log((jsonString)); - callback(JSON.stringify(jsonString)); - }); - gunzip.on('error', function (e) { - console.log(e); - }); - } - else - { - var b = processBody(body,url, _id, _ver); - if (res != null) - { - res.render('grabbed'); - } - } - - } else - { - var b = processBody(body,url, _id, _ver); - if (res != null) - { - res.render('grabbed',{data:b}); - } + var gunzip = zlib.createGunzip(); + var jsonString = ''; + resp.pipe(gunzip); + gunzip.on('data', function(chunk) { + jsonString += chunk; + }); + gunzip.on('end', function() { + console.log((jsonString)); + callback(JSON.stringify(jsonString)); + }); + gunzip.on('error', function(e) { + console.log(e); + }); + } + else { + var b = processBody(body, url, _id, _ver); + if (res != null) { + res.render('grabbed'); } + } - }); + } + else { + var b = processBody(body, url, _id, _ver); + if (res != null) { + res.render('grabbed', {data: b}); + } + } + + }); } -router.get('/list', function (req, res) { - logger.debug('list..'); +router.get('/list', function(req, res) { + logger.debug('list..'); - dbCouch.view('titles','titles',function(err, body) { + dbCouch.view('titles', 'titles', function(err, body) { if (!err) { var outJSON = []; body.rows.forEach(function(doc) { - outJSON.push({id:doc.id, title:doc.value }) + outJSON.push({id: doc.id, title: doc.value}) }); //logger.debug(util.inspect(body)); res.writeHead(200, {"ContentType": "application/json"}); res.end(JSON.stringify({list: outJSON})); - } else - { + } + else { res.writeHead(500, {"ContentType": "application/json"}); res.end(JSON.stringify({})); } }); }); -router.get('/entry/:id', function (req, res) { +router.get('/entry/:id', function(req, res) { logger.debug('entry..'); logger.debug(req.params.id); - dbCouch.get(req.params.id,function(err, body) { + dbCouch.get(req.params.id, function(err, body) { if (!err) { var outJSON = {}; @@ -276,8 +293,8 @@ router.get('/entry/:id', function (req, res) { res.writeHead(200, {"ContentType": "application/json"}); res.end(JSON.stringify(outJSON)); - } else - { + } + else { res.writeHead(500, {"ContentType": "application/json"}); res.end(JSON.stringify({})); } @@ -285,24 +302,24 @@ router.get('/entry/:id', function (req, res) { }); -router.post('/add', function (req, res) { - logger.debug('add entry..'); +router.post('/add', function(req, res) { + logger.debug('add entry..'); - var t = req.body; - if (t.hasOwnProperty('url')) { - var url = JSON.parse(t.url.toString()); - logger.debug(url); - busEmitter.emit("getBookmark", t); - } - else { - logger.error('No data block!'); - } - res.writeHead(200, {"ContentType": "application/json"}); - res.end(JSON.stringify({adding: url})); + var t = req.body; + if (t.hasOwnProperty('url')) { + var url = JSON.parse(t.url.toString()); + logger.debug(url); + busEmitter.emit("getBookmark", t); + } + else { + logger.error('No data block!'); + } + res.writeHead(200, {"ContentType": "application/json"}); + res.end(JSON.stringify({adding: url})); }); -router.post('/redo', function (req, res) { +router.post('/redo', function(req, res) { logger.debug('redoing entry..'); var t = req.body; @@ -320,11 +337,9 @@ router.post('/redo', function (req, res) { }); - - -router.get('/new', function (req, res) { +router.get('/new', function(req, res) { logger.debug('Save new'); - busEmitter.emit("getBookmarkRes", req.query.url ,res); + busEmitter.emit("getBookmarkRes", req.query.url, res); });