martin/keeper
martin/keeper
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Doing some work on version 2

Browse Source
This commit is contained in:
martind2000 2017-06-27 17:11:07 +01:00
parent 8657f7362f
commit b33aebfecc
6 changed files with 356 additions and 234 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

155
.eslintrc Normal file
View File

@ -0,0 +1,155 @@
{
"parser": "babel-eslint",
"rules": {
/**
* Strict mode
*/
// babel inserts "use strict"; for us
"strict": [2, "never"], // http://eslint.org/docs/rules/strict
/**
* ES6
*/
"no-var": 2, // http://eslint.org/docs/rules/no-var
"prefer-const": 0, // http://eslint.org/docs/rules/prefer-const
/**
* Variables
*/
"no-shadow": 2, // http://eslint.org/docs/rules/no-shadow
"no-shadow-restricted-names": 2, // http://eslint.org/docs/rules/no-shadow-restricted-names
"no-unused-vars": [2, { // http://eslint.org/docs/rules/no-unused-vars
"vars": "local",
"args": "none"
}],
"no-use-before-define": 0, // http://eslint.org/docs/rules/no-use-before-define
/**
* Possible errors
*/
"comma-dangle": [2, "never"], // http://eslint.org/docs/rules/comma-dangle
"no-cond-assign": [2, "always"], // http://eslint.org/docs/rules/no-cond-assign
"no-console": 1, // http://eslint.org/docs/rules/no-console
"no-debugger": 1, // http://eslint.org/docs/rules/no-debugger
"no-alert": 1, // http://eslint.org/docs/rules/no-alert
"no-constant-condition": 1, // http://eslint.org/docs/rules/no-constant-condition
"no-dupe-keys": 2, // http://eslint.org/docs/rules/no-dupe-keys
"no-duplicate-case": 2, // http://eslint.org/docs/rules/no-duplicate-case
"no-empty": 2, // http://eslint.org/docs/rules/no-empty
"no-ex-assign": 2, // http://eslint.org/docs/rules/no-ex-assign
"no-extra-boolean-cast": 0, // http://eslint.org/docs/rules/no-extra-boolean-cast
"no-extra-semi": 2, // http://eslint.org/docs/rules/no-extra-semi
"no-func-assign": 2, // http://eslint.org/docs/rules/no-func-assign
"no-inner-declarations": 2, // http://eslint.org/docs/rules/no-inner-declarations
"no-invalid-regexp": 2, // http://eslint.org/docs/rules/no-invalid-regexp
"no-irregular-whitespace": 2, // http://eslint.org/docs/rules/no-irregular-whitespace
"no-obj-calls": 2, // http://eslint.org/docs/rules/no-obj-calls
"no-sparse-arrays": 2, // http://eslint.org/docs/rules/no-sparse-arrays
"no-unreachable": 2, // http://eslint.org/docs/rules/no-unreachable
"use-isnan": 2, // http://eslint.org/docs/rules/use-isnan
"block-scoped-var": 0, // http://eslint.org/docs/rules/block-scoped-var
/**
* Best practices
*/
"consistent-return": 0, // http://eslint.org/docs/rules/consistent-return
"curly": [2, "multi-line"], // http://eslint.org/docs/rules/curly
"default-case": 2, // http://eslint.org/docs/rules/default-case
"dot-notation": [2, { // http://eslint.org/docs/rules/dot-notation
"allowKeywords": true
}],
"eqeqeq": 2, // http://eslint.org/docs/rules/eqeqeq
"guard-for-in": 0, // http://eslint.org/docs/rules/guard-for-in
"no-caller": 2, // http://eslint.org/docs/rules/no-caller
"no-else-return": 2, // http://eslint.org/docs/rules/no-else-return
"no-eq-null": 2, // http://eslint.org/docs/rules/no-eq-null
"no-eval": 2, // http://eslint.org/docs/rules/no-eval
"no-extend-native": 2, // http://eslint.org/docs/rules/no-extend-native
"no-extra-bind": 2, // http://eslint.org/docs/rules/no-extra-bind
"no-fallthrough": 2, // http://eslint.org/docs/rules/no-fallthrough
"no-floating-decimal": 2, // http://eslint.org/docs/rules/no-floating-decimal
"no-implied-eval": 2, // http://eslint.org/docs/rules/no-implied-eval
"no-lone-blocks": 2, // http://eslint.org/docs/rules/no-lone-blocks
"no-loop-func": 2, // http://eslint.org/docs/rules/no-loop-func
"no-multi-str": 2, // http://eslint.org/docs/rules/no-multi-str
"no-native-reassign": 2, // http://eslint.org/docs/rules/no-native-reassign
"no-new": 2, // http://eslint.org/docs/rules/no-new
"no-new-func": 2, // http://eslint.org/docs/rules/no-new-func
"no-new-wrappers": 2, // http://eslint.org/docs/rules/no-new-wrappers
"no-octal": 2, // http://eslint.org/docs/rules/no-octal
"no-octal-escape": 2, // http://eslint.org/docs/rules/no-octal-escape
"no-param-reassign": 0, // http://eslint.org/docs/rules/no-param-reassign
"no-proto": 2, // http://eslint.org/docs/rules/no-proto
"no-redeclare": 2, // http://eslint.org/docs/rules/no-redeclare
// "no-return-assign": 2, // http://eslint.org/docs/rules/no-return-assign
"no-script-url": 2, // http://eslint.org/docs/rules/no-script-url
"no-self-compare": 2, // http://eslint.org/docs/rules/no-self-compare
"no-sequences": 2, // http://eslint.org/docs/rules/no-sequences
"no-throw-literal": 2, // http://eslint.org/docs/rules/no-throw-literal
"no-with": 2, // http://eslint.org/docs/rules/no-with
"radix": 2, // http://eslint.org/docs/rules/radix
"vars-on-top": 2, // http://eslint.org/docs/rules/vars-on-top
"wrap-iife": [2, "any"], // http://eslint.org/docs/rules/wrap-iife
"yoda": 2, // http://eslint.org/docs/rules/yoda
/**
* Style
*/
"indent": [2, 2], // http://eslint.org/docs/rules/indent
"brace-style": [2, // http://eslint.org/docs/rules/brace-style
"1tbs", {
"allowSingleLine": true
}],
"quotes": [
2, "single", "avoid-escape" // http://eslint.org/docs/rules/quotes
],
"camelcase": [2, { // http://eslint.org/docs/rules/camelcase
"properties": "never"
}],
"comma-spacing": [2, { // http://eslint.org/docs/rules/comma-spacing
"before": false,
"after": true
}],
"comma-style": [2, "last"], // http://eslint.org/docs/rules/comma-style
"eol-last": 2, // http://eslint.org/docs/rules/eol-last
"func-names": 0, // http://eslint.org/docs/rules/func-names
"key-spacing": [2, { // http://eslint.org/docs/rules/key-spacing
"beforeColon": false,
"afterColon": true
}],
"new-cap": [2, { // http://eslint.org/docs/rules/new-cap
"newIsCap": true
}],
"no-multiple-empty-lines": [2, { // http://eslint.org/docs/rules/no-multiple-empty-lines
"max": 2
}],
"no-nested-ternary": 2, // http://eslint.org/docs/rules/no-nested-ternary
"no-new-object": 2, // http://eslint.org/docs/rules/no-new-object
"no-spaced-func": 2, // http://eslint.org/docs/rules/no-spaced-func
"no-trailing-spaces": 2, // http://eslint.org/docs/rules/no-trailing-spaces
"no-extra-parens": [2, "functions"], // http://eslint.org/docs/rules/no-extra-parens
"no-underscore-dangle": 0, // http://eslint.org/docs/rules/no-underscore-dangle
"one-var": [2, "never"], // http://eslint.org/docs/rules/one-var
"padded-blocks": [2, "never"], // http://eslint.org/docs/rules/padded-blocks
"semi": [2, "always"], // http://eslint.org/docs/rules/semi
"semi-spacing": [2, { // http://eslint.org/docs/rules/semi-spacing
"before": false,
"after": true
}],
"keyword-spacing": [2, { // http://eslint.org/docs/rules/keyword-spacing
"before": true,
"after": true
}],
"space-before-blocks": 2, // http://eslint.org/docs/rules/space-before-blocks
"space-before-function-paren": [2, "never"], // http://eslint.org/docs/rules/space-before-function-paren
"space-infix-ops": 2, // http://eslint.org/docs/rules/space-infix-ops
"spaced-comment": [0, "always", { // http://eslint.org/docs/rules/spaced-comment
"exceptions": ["*"],
"markers": ["*"]
}]
},
"env": {
"node": true,
"browser": true
}
}

25
keeper-serverV2.js
View File

@ -1,14 +1,14 @@
var express = require('express'), path = require('path'), http = require('http'), const express = require('express'), path = require('path'), http = require('http'),
favicon = require('serve-favicon'), favicon = require('serve-favicon'),
logger = require('morgan'), logger = require('morgan'),
cookieParser = require('cookie-parser'), cookieParser = require('cookie-parser'),
bodyParser = require('body-parser'), bodyParser = require('body-parser'),
keeper = require('./server/keeperV2') keeper = require('./server/keeperV2')
; ;
var app = express(); const app = express();
app.set('port', process.env.PORT || 8026); app.set('port', process.env.PORT || 8026);
app.engine('html', require('ejs').renderFile); app.engine('html', require('ejs').renderFile);
@ -18,7 +18,7 @@ app.set('view engine', 'html');
app.use(logger('dev')); app.use(logger('dev'));
app.use(bodyParser.json()); app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true })); app.use(bodyParser.urlencoded({extended: true}));
app.use(cookieParser()); app.use(cookieParser());
//app.use(express.static(path.join(__dirname, 'dist'))); //app.use(express.static(path.join(__dirname, 'dist')));
@ -30,8 +30,9 @@ app.use('/', keeper);
* create the server * create the server
*/ */
app.listen(app.get('port'), function () { app.listen(app.get('port'), function() {
console.log('Keeper Server listening on ' + app.get('port')); console.log('Keeper Server listening on ' + app.get('port'));
});/** });
/**
* Created by Martin on 22/02/2016. * Created by Martin on 22/02/2016.
*/ */

2
package.json
View File

@ -24,7 +24,9 @@
"url": "^0.11.0" "url": "^0.11.0"
}, },
"devDependencies": { "devDependencies": {
"babel-eslint": "^7.2.3",
"del": "^2.2.0", "del": "^2.2.0",
"eslint": "^4.1.1",
"gulp": "^3.9.1", "gulp": "^3.9.1",
"gulp-autoprefixer": "^3.1.0", "gulp-autoprefixer": "^3.1.0",
"gulp-cache": "^0.4.2", "gulp-cache": "^0.4.2",

28
server/dbcloudant.js Normal file
View File

@ -0,0 +1,28 @@
/**
* Created by mdonnel on 27/06/2017.
*/
/*
We've moved to cloudant through IBM Bluemix for the database
https://25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix.cloudant.com/dashboard.html#usage
*/
const credentials = {
'username': '25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix',
'password': '8e417af1b0462ca55726848846cc6b8696fc76defe9d1864cbc334be59549e0c',
'host': '25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix.cloudant.com',
'port': 443,
'url': 'https://25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix:8e417af1b0462ca55726848846cc6b8696fc76defe9d1864cbc334be59549e0c@25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix.cloudant.com',
'database': 'keeper'
};
const cloudant = require('cloudant');
const cloudantConnection = cloudant({account: credentials.username, password: credentials.password});
exports.dbcloudant = cloudantConnection.db.use(credentials.database);

374
server/keeperV2.js
View File

@ -1,56 +1,36 @@
'use strict';
/** /**
* Created by Martin on 22/02/2016. * Created by Martin on 22/02/2016.
*/ */
var express = require('express'); let express = require('express');
var http = require('http'), request = require('request'), cheerio = require( let http = require('http'), request = require('request'), cheerio = require('cheerio'), util = require('util');
'cheerio'), util = require('util'); let jsonfile = require('jsonfile'), fs = require('fs'), STRING = require('string');
var jsonfile = require('jsonfile'), fs = require('fs'), STRING = require( let converter = require('html-to-markdown');
'string'); let zlib = require('zlib');
var converter = require('html-to-markdown'); let log4js = require('log4js');
var zlib = require('zlib'); let logger = log4js.getLogger();
var log4js = require('log4js'); let URL = require('url');
var logger = log4js.getLogger();
var URL = require('url');
var router = express.Router(); let router = express.Router();
var EventEmitter = require('events'); let EventEmitter = require('events');
//var nano = require('nano')('http://martind2000:1V3D4m526i@localhost:5984'); //var nano = require('nano')('http://martind2000:1V3D4m526i@localhost:5984');
var busEmitter = new EventEmitter(); let busEmitter = new EventEmitter();
const dbCloudant = require('./dbcloudant').dbcloudant;
var db_name = 'keeper';
//var dbCouch = nano.use(db_name); //var dbCouch = nano.use(db_name);
/*
We've moved to cloudant through IBM Bluemix for the database
https://25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix.cloudant.com/dashboard.html#usage
*/
var credentials = {
"username": "25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix",
"password": "8e417af1b0462ca55726848846cc6b8696fc76defe9d1864cbc334be59549e0c",
"host": "25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix.cloudant.com",
"port": 443,
"url": "https://25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix:8e417af1b0462ca55726848846cc6b8696fc76defe9d1864cbc334be59549e0c@25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix.cloudant.com",
"database" : "keeper"
};
var Cloudant = require('cloudant');
var cloudant = Cloudant({account:credentials.username, password:credentials.password});
var dbCloudant = cloudant.db.use(credentials.database);
var jsonFile = __dirname + '/' + 'output.json';
var bodyfile = __dirname + '/' + 'body.html'; let jsonFile = __dirname + '/' + 'output.json';
var htmlfile = __dirname + '/' + 'testoutput.html'; let bodyfile = __dirname + '/' + 'body.html';
var generics = [ let htmlfile = __dirname + '/' + 'testoutput.html';
let generics = [
'ARTICLE', 'ARTICLE',
'div.content_column', 'div.content_column',
'div.post', 'div.post',
@ -59,33 +39,30 @@ var generics = [
'div.content.body' 'div.content.body'
]; ];
var specialHandlers = [{ let specialHandlers = [{
url: 'www.reddit.com', fn: function(body, url) { url: 'www.reddit.com', fn: function(body, url) {
return doReddit(body, url); return doReddit(body, url);
} }
}, },
{ {
url: 'developer.android.com', fn: function(body, url) { url: 'developer.android.com', fn: function(body, url) {
return doAndroidDeveloper(body, url); return doAndroidDeveloper(body, url);
}
},
{
url: 'www.engadget.com', fn: function(body, url) {
return doEngadget(body, url);
}
} }
},
{
url: 'www.engadget.com', fn: function(body, url) {
return doEngadget(body, url);
}
}
]; ];
function cleaner(b) { function cleaner(b) {
var _b = b; let _b = b;
var unwanted = [ let unwanted = [
'LINK', 'LINK',
'META', 'META',
'TITLE', 'TITLE',
@ -117,7 +94,7 @@ function cleaner(b) {
]; ];
for (var i = 0; i < unwanted.length; i++) { for (let i = 0; i < unwanted.length; i++) {
_b.find(unwanted[i]).remove(); _b.find(unwanted[i]).remove();
} }
@ -127,7 +104,7 @@ function cleaner(b) {
function insertBookmark(obj) { function insertBookmark(obj) {
logger.debug('Inserting into couch...'); logger.debug('Inserting into couch...');
logger.info(util.inspect(obj)); logger.info(util.inspect(obj));
// dbCouch.insert(obj, function(err, body, header) { // dbCouch.insert(obj, function(err, body, header) {
dbCloudant.insert(obj, function(err, body, header) { dbCloudant.insert(obj, function(err, body, header) {
if (err) { if (err) {
logger.error('Error inserting into couch'); logger.error('Error inserting into couch');
@ -139,7 +116,7 @@ function insertBookmark(obj) {
function updateBookmark(obj, _id, _rev) { function updateBookmark(obj, _id, _rev) {
logger.debug('Updating couch...'); logger.debug('Updating couch...');
var _obj = obj; let _obj = obj;
_obj._id = _id; _obj._id = _id;
_obj._rev = _rev; _obj._rev = _rev;
@ -148,49 +125,46 @@ function updateBookmark(obj, _id, _rev) {
if (err) { if (err) {
logger.error('Error updating into couch'); logger.error('Error updating into couch');
return; return;
} else {
logger.info('I think we updated ok...');
busEmitter.emit('updateTagsDB');
} }
logger.info('I think we updated ok...');
busEmitter.emit('updateTagsDB');
}); });
logger.debug('Update done..'); logger.debug('Update done..');
} }
var doInsertBookmark = (obj) => { let doInsertBookmark = (obj) => {
// Logger.info('sendSocket: ' + JSON.stringify(obj)); // Logger.info('sendSocket: ' + JSON.stringify(obj));
insertBookmark(obj); insertBookmark(obj);
}; };
var doUpdateBookmark = (obj, _id, _rev) => { let doUpdateBookmark = (obj, _id, _rev) => {
// Logger.info('sendSocket: ' + JSON.stringify(obj)); // Logger.info('sendSocket: ' + JSON.stringify(obj));
updateBookmark(obj, _id, _rev); updateBookmark(obj, _id, _rev);
}; };
var doGetBookmark = (obj) => { let doGetBookmark = (obj) => {
// Logger.info('sendSocket: ' + JSON.stringify(obj)); // Logger.info('sendSocket: ' + JSON.stringify(obj));
genericGrab(obj); genericGrab(obj);
}; };
var doGetBookmarkRedo = (obj) => { let doGetBookmarkRedo = (obj) => {
// Logger.info('sendSocket: ' + JSON.stringify(obj)); // Logger.info('sendSocket: ' + JSON.stringify(obj));
genericGrab(obj); genericGrab(obj);
}; };
var doGetBookmarkRes = (url, res) => { let doGetBookmarkRes = (url, res) => {
logger.debug('doGetBookmarkRes'); logger.debug('doGetBookmarkRes');
// Logger.info('sendSocket: ' + JSON.stringify(obj)); // Logger.info('sendSocket: ' + JSON.stringify(obj));
genericGrab(url, res); genericGrab(url, res);
}; };
var doUpdateTagsDB = () => { let doUpdateTagsDB = () => {
logger.debug('Update the tags database...'); logger.debug('Update the tags database...');
// dbCouch.view('getAllTags', 'getAllTags', function(err, body) { // dbCouch.view('getAllTags', 'getAllTags', function(err, body) {
dbCloudant.view('getAllTags', 'getAllTags', function(err, body) { dbCloudant.view('getAllTags', 'getAllTags', function(err, body) {
var masterList = []; let masterList = [];
if (!err) { if (!err) {
body.rows.forEach(function(doc) { body.rows.forEach(function(doc) {
masterList = masterList.concat(doc.value); masterList = masterList.concat(doc.value);
}); });
@ -202,29 +176,25 @@ var doUpdateTagsDB = () => {
dbCloudant.view('taglist', 'taglist', function(err, body) { dbCloudant.view('taglist', 'taglist', function(err, body) {
// Logger.debug(body); // Logger.debug(body);
if (!err) { if (!err) {
let outJSON = {};
var outJSON = {};
body.rows.forEach(function(doc) { body.rows.forEach(function(doc) {
doSaveTagsDB(doc.value, masterList); doSaveTagsDB(doc.value, masterList);
}); });
} else { } else {
logger.error('NO TAG LIST EXISTS'); logger.error('NO TAG LIST EXISTS');
} }
}); });
} else { } else {
} }
}); });
}; };
var doSaveTagsDB = (orig, newList) => { var doSaveTagsDB = (orig, newList) => {
logger.debug('doSaveTagsDB'); logger.debug('doSaveTagsDB');
var _obj = orig; let _obj = orig;
_obj.taglist = newList; _obj.taglist = newList;
@ -233,10 +203,8 @@ var doSaveTagsDB = (orig, newList) => {
if (err) { if (err) {
logger.error('Error updating into couch'); logger.error('Error updating into couch');
return; return;
} else {
logger.info('Updated the tags list...');
} }
logger.info('Updated the tags list...');
}); });
}; };
@ -252,14 +220,14 @@ busEmitter.on('saveTagsDB', doSaveTagsDB);
function doEngadget(body, url) { function doEngadget(body, url) {
logger.info('GRABBING Engadget'); logger.info('GRABBING Engadget');
var obj = {}, tdihbody, i, urlObj, urlPrefix; let obj = {}, tdihbody, i, urlObj, urlPrefix;
var $ = cheerio.load(body); let $ = cheerio.load(body);
var title = $('TITLE').text(); let title = $('TITLE').text();
tdihbody = $('DIV#page_body'); tdihbody = $('DIV#page_body');
logger.debug('Length:' , tdihbody.length); logger.debug('Length:', tdihbody.length);
tdihbody = cleaner(tdihbody); tdihbody = cleaner(tdihbody);
logger.debug('Title: ', title); logger.debug('Title: ', title);
@ -284,10 +252,8 @@ function doEngadget(body, url) {
$(this).attr('src', s); $(this).attr('src', s);
} }
}); });
} } catch (e) {
catch (e) {
logger.error(e); logger.error(e);
} }
@ -304,10 +270,10 @@ function doEngadget(body, url) {
function doAndroidDeveloper(body, url) { function doAndroidDeveloper(body, url) {
logger.info('GRABBING AndroidDeveloper'); logger.info('GRABBING AndroidDeveloper');
var obj = {}, tdihbody, i, urlObj, urlPrefix; let obj = {}, tdihbody, i, urlObj, urlPrefix;
var $ = cheerio.load(body); let $ = cheerio.load(body);
var title = $('TITLE').text(); let title = $('TITLE').text();
tdihbody = $('DIV.jd-descr'); tdihbody = $('DIV.jd-descr');
@ -336,10 +302,8 @@ function doAndroidDeveloper(body, url) {
$(this).attr('src', s); $(this).attr('src', s);
} }
}); });
} } catch (e) {
catch (e) {
logger.error(e); logger.error(e);
} }
@ -356,15 +320,14 @@ function doAndroidDeveloper(body, url) {
function doReddit(body, url) { function doReddit(body, url) {
logger.info('GRABBING REDDIT'); logger.info('GRABBING REDDIT');
var obj = {}, tdihbody, i, urlObj, urlPrefix; let obj = {}, tdihbody, i, urlObj, urlPrefix;
var $ = cheerio.load(body); let $ = cheerio.load(body);
var title = $('TITLE').text(); let title = $('TITLE').text();
tdihbody = $('DIV.entry'); tdihbody = $('DIV.entry');
tdihbody.find('A.thumbnail').each(function(i, elem) { tdihbody.find('A.thumbnail').each(function(i, elem) {
logger.warn($(this)); logger.warn($(this));
}); });
@ -386,12 +349,11 @@ function doReddit(body, url) {
} }
function genericProcessor(body, url) { function genericProcessor(body, url) {
logger.info('USING DEFAULT PROCESSOR'); logger.info('USING DEFAULT PROCESSOR');
var obj = {}, tdihbody, i, urlObj, urlPrefix; let obj = {}, tdihbody, i, urlObj, urlPrefix;
var $ = cheerio.load(body); let $ = cheerio.load(body);
var title = $('TITLE').text(); let title = $('TITLE').text();
i = 0; i = 0;
@ -406,16 +368,13 @@ function genericProcessor(body, url) {
logger.debug(tdihbody.length); logger.debug(tdihbody.length);
tdihbody = cleaner(tdihbody); tdihbody = cleaner(tdihbody);
logger.debug(title); logger.debug(title);
} else { } else {
logger.warn('Using whole body'); logger.warn('Using whole body');
// Bah. nothing to reduce so just grab the body, tidy it and use that // Bah. nothing to reduce so just grab the body, tidy it and use that
tdihbody = $('BODY'); tdihbody = $('BODY');
if (tdihbody.length === 0) { if (tdihbody.length === 0) {
tdihbody = $(':root'); tdihbody = $(':root');
} }
logger.debug(tdihbody.length); logger.debug(tdihbody.length);
@ -447,10 +406,8 @@ function genericProcessor(body, url) {
$(this).attr('src', s); $(this).attr('src', s);
} }
}); });
} } catch (e) {
catch (e) {
logger.error(e); logger.error(e);
} }
@ -465,10 +422,7 @@ function genericProcessor(body, url) {
return obj; return obj;
} }
function processBody(body, url, _id, _rev) { function processBody(body, url, _id, _rev) {
let obj = {}, i, urlObj, urlPrefix;
var obj = {}, i, urlObj, urlPrefix;
// Try to find a body to grab // Try to find a body to grab
@ -477,17 +431,17 @@ function processBody(body, url, _id, _rev) {
logger.debug('host:', urlObj.host); logger.debug('host:', urlObj.host);
var flag; let flag;
for (i = 0;i < specialHandlers.length;i++) { for (i = 0; i < specialHandlers.length; i++) {
if (urlObj.host === specialHandlers[i].url) { if (urlObj.host === specialHandlers[i].url) {
flag = true; flag = true;
obj = specialHandlers[i].fn(body,url); obj = specialHandlers[i].fn(body, url);
} }
} }
if (!flag) { if (!flag) {
// Do generic processing // Do generic processing
obj = genericProcessor(body,url); obj = genericProcessor(body, url);
} }
// Logger.warn(obj.reduced); // Logger.warn(obj.reduced);
@ -506,11 +460,9 @@ function processBody(body, url, _id, _rev) {
} }
return obj; return obj;
} }
function genericGrab(obj, res) { function genericGrab(obj, res) {
let url, _id = null, _ver = null;
var url, _id = null, _ver = null;
if (typeof obj === 'string') { if (typeof obj === 'string') {
logger.info(obj); logger.info(obj);
@ -527,7 +479,7 @@ function genericGrab(obj, res) {
logger.info(_id); logger.info(_id);
logger.info(_ver); logger.info(_ver);
var options = { let options = {
url: url, url: url,
headers: { headers: {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36' 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36'
@ -538,17 +490,15 @@ function genericGrab(obj, res) {
}; };
request(options, function(err, resp, body) { request(options, function(err, resp, body) {
if (err) if (err) {throw err;}
throw err;
if (resp.headers.hasOwnProperty('content-encoding')) { if (resp.headers.hasOwnProperty('content-encoding')) {
logger.warn('content-encoding'); logger.warn('content-encoding');
if (resp.headers['content-encoding'] == 'gzip') { if (resp.headers['content-encoding'] == 'gzip') {
// to test http://chaosinthekitchen.com/2009/07/lime-and-coconut-chicken/ // to test http://chaosinthekitchen.com/2009/07/lime-and-coconut-chicken/
var gunzip = zlib.createGunzip(); let gunzip = zlib.createGunzip();
var jsonString = ''; let jsonString = '';
resp.pipe(gunzip); resp.pipe(gunzip);
gunzip.on('data', function(chunk) { gunzip.on('data', function(chunk) {
jsonString += chunk; jsonString += chunk;
@ -566,14 +516,12 @@ function genericGrab(obj, res) {
res.render('grabbed'); res.render('grabbed');
} }
} }
} else { } else {
var b = processBody(body, url, _id, _ver); var b = processBody(body, url, _id, _ver);
if (res != null) { if (res != null) {
res.render('grabbed', {data: b}); res.render('grabbed', {data: b});
} }
} }
}); });
} }
@ -581,24 +529,21 @@ router.get('/pocket', function(req, res) {
logger.debug('list..'); logger.debug('list..');
// dbCouch.view('pocketList', 'pocketList', function(err, body) { // dbCouch.view('pocketList', 'pocketList', function(err, body) {
dbCloudant.view('pocketList', 'pocketList', function(err, body) { dbCloudant.view('pocketList', 'pocketList', function(err, body) {
if (!err) { if (!err) {
let outJSON = [];
var outJSON = [];
body.rows.forEach(function(doc) { body.rows.forEach(function(doc) {
var obj = {id: doc.id, entry: doc.value}; let obj = {id: doc.id, entry: doc.value};
console.log(typeof obj.entry.tn); console.log(typeof obj.entry.tn);
if (typeof obj.entry.tn === 'string') { if (typeof obj.entry.tn === 'string') {
console.log('its a string:', typeof obj.entry.tn) console.log('its a string:', typeof obj.entry.tn);
obj.entry.tn = 'http://image.silvrtree.co.uk/100,fit,q80/' + obj.entry.tn; obj.entry.tn = 'http://image.silvrtree.co.uk/100,fit,q80/' + obj.entry.tn;
} else { } else {
obj.entry.tn = 'gfx/fm.png'; obj.entry.tn = 'gfx/fm.png';
} }
outJSON.push(obj); outJSON.push(obj);
}); });
logger.debug(util.inspect(body)); logger.debug(util.inspect(body));
@ -612,15 +557,13 @@ router.get('/pocket', function(req, res) {
}); });
router.get('/list', function(req, res) { router.get('/list', function(req, res) {
logger.debug('list..'); logger.debug('list..');
//dbCouch.view('titles', 'titles', function(err, body) { //dbCouch.view('titles', 'titles', function(err, body) {
dbCloudant.view('titles', 'titles', function(err, body) { dbCloudant.view('titles', 'titles', function(err, body) {
if (!err) { if (!err) {
let outJSON = [];
var outJSON = [];
body.rows.forEach(function(doc) { body.rows.forEach(function(doc) {
outJSON.push({id: doc.id, title: doc.value}); outJSON.push({id: doc.id, title: doc.value});
}); });
@ -628,7 +571,6 @@ router.get('/list', function(req, res) {
//Logger.debug(util.inspect(body)); //Logger.debug(util.inspect(body));
res.writeHead(200, {ContentType: 'application/json'}); res.writeHead(200, {ContentType: 'application/json'});
res.end(JSON.stringify({list: outJSON})); res.end(JSON.stringify({list: outJSON}));
} else { } else {
res.writeHead(500, {ContentType: 'application/json'}); res.writeHead(500, {ContentType: 'application/json'});
res.end(JSON.stringify({})); res.end(JSON.stringify({}));
@ -644,8 +586,7 @@ router.get('/entry/:id', function(req, res) {
//dbCouch.get(req.params.id, function(err, body) { //dbCouch.get(req.params.id, function(err, body) {
dbCloudant.get(req.params.id, function(err, body) { dbCloudant.get(req.params.id, function(err, body) {
if (!err) { if (!err) {
let outJSON = {};
var outJSON = {};
outJSON._id = body._id; outJSON._id = body._id;
outJSON._rev = body._rev; outJSON._rev = body._rev;
outJSON.title = body.title; outJSON.title = body.title;
@ -656,85 +597,78 @@ router.get('/entry/:id', function(req, res) {
//Logger.debug(util.inspect(body)); //Logger.debug(util.inspect(body));
res.writeHead(200, {ContentType: 'application/json'}); res.writeHead(200, {ContentType: 'application/json'});
res.end(JSON.stringify(outJSON)); res.end(JSON.stringify(outJSON));
} else { } else {
res.writeHead(500, {ContentType: 'application/json'}); res.writeHead(500, {ContentType: 'application/json'});
res.end(JSON.stringify({})); res.end(JSON.stringify({}));
} }
}); });
}); });
router.route('/tags') router.route('/tags')
.get(function(req, res, next) { .get(function(req, res, next) {
logger.debug('tag list..'); logger.debug('tag list..');
logger.debug(req.params.id); logger.debug(req.params.id);
//dbCouch.view('taglist', 'taglist', function(err, body) { //dbCouch.view('taglist', 'taglist', function(err, body) {
dbCloudant.view('taglist', 'taglist', function(err, body) { dbCloudant.view('taglist', 'taglist', function(err, body) {
if (!err) { if (!err) {
logger.debug(body); logger.debug(body);
var outJSON = []; let outJSON = [];
body.rows.forEach(function(doc) { body.rows.forEach(function(doc) {
logger.info(doc.value.taglist); logger.info(doc.value.taglist);
if (doc.value[0] == req.params.id) { if (doc.value[0] == req.params.id) {
outJSON = doc.value.taglist.sort(); outJSON = doc.value.taglist.sort();
} }
}); });
//Logger.debug(util.inspect(body)); //Logger.debug(util.inspect(body));
res.writeHead(200, {ContentType: 'application/json'}); res.writeHead(200, {ContentType: 'application/json'});
res.end(JSON.stringify({list: outJSON})); res.end(JSON.stringify({list: outJSON}));
} else {
logger.error(err);
res.writeHead(500, {ContentType: 'application/json'});
res.end(JSON.stringify({}));
}
});
}).post(function(req, res, next) {
let t = req.body;
console.log(t);
} else { logger.info('regetting:' + req.body._id);
logger.error(err);
res.writeHead(500, {ContentType: 'application/json'}); //dbCouch.get(req.body._id, function(err, body) {
res.end(JSON.stringify({})); dbCloudant.get(req.body._id, function(err, body) {
} if (!err) {
let obj = {};
obj.url = body.url;
obj.html = body.html;
obj.reduced = body.reduced;
obj.title = body.title;
obj.tags = req.body.tags;
logger.info('Updating...');
busEmitter.emit('updateBookmarkData', obj, body._id, body._rev, res);
let outJSON = {};
outJSON._id = body._id;
outJSON._rev = body._rev;
outJSON.title = body.title;
outJSON.reduced = body.reduced;
outJSON.url = body.url;
outJSON.tags = req.body.tags;
//Logger.debug(util.inspect(body));
res.writeHead(200, {ContentType: 'application/json'});
res.end(JSON.stringify(outJSON));
} else {
res.writeHead(500, {ContentType: 'application/json'});
res.end(JSON.stringify({}));
}
});
}); });
}).post(function(req, res, next) {
var t = req.body;
console.log(t);
logger.info('regetting:' + req.body._id);
//dbCouch.get(req.body._id, function(err, body) {
dbCloudant.get(req.body._id, function(err, body) {
if (!err) {
var obj = {};
obj.url = body.url;
obj.html = body.html;
obj.reduced = body.reduced;
obj.title = body.title;
obj.tags = req.body.tags;
logger.info('Updating...');
busEmitter.emit('updateBookmarkData', obj, body._id, body._rev, res);
var outJSON = {};
outJSON._id = body._id;
outJSON._rev = body._rev;
outJSON.title = body.title;
outJSON.reduced = body.reduced;
outJSON.url = body.url;
outJSON.tags = req.body.tags;
//Logger.debug(util.inspect(body));
res.writeHead(200, {ContentType: 'application/json'});
res.end(JSON.stringify(outJSON));
} else {
res.writeHead(500, {ContentType: 'application/json'});
res.end(JSON.stringify({}));
}
});
});
router.get('/tags/:id', function(req, res) { router.get('/tags/:id', function(req, res) {
logger.debug('entry..'); logger.debug('entry..');
@ -745,31 +679,29 @@ router.get('/tags/:id', function(req, res) {
dbCloudant.view('getTagByKey', 'getTagByKey', function(err, body) { dbCloudant.view('getTagByKey', 'getTagByKey', function(err, body) {
if (!err) { if (!err) {
// Logger.debug(body); // Logger.debug(body);
var outJSON = []; let outJSON = [];
body.rows.forEach(function(doc) { body.rows.forEach(function(doc) {
// Logger.debug(doc); // Logger.debug(doc);
if (doc.value[0] == req.params.id) { if (doc.value[0] == req.params.id) {
outJSON.push({id: doc.id, title: doc.value[1]}) outJSON.push({id: doc.id, title: doc.value[1]});
} }
}); });
//Logger.debug(util.inspect(body)); //Logger.debug(util.inspect(body));
res.writeHead(200, {ContentType: 'application/json'}); res.writeHead(200, {ContentType: 'application/json'});
res.end(JSON.stringify({list: outJSON})); res.end(JSON.stringify({list: outJSON}));
} else { } else {
logger.error(err); logger.error(err);
res.writeHead(500, {ContentType: 'application/json'}); res.writeHead(500, {ContentType: 'application/json'});
res.end(JSON.stringify({})); res.end(JSON.stringify({}));
} }
}); });
}); });
router.post('/add', function(req, res) { router.post('/add', function(req, res) {
logger.debug('add entry..'); logger.debug('add entry..');
var t = req.body; let t = req.body;
if (t.hasOwnProperty('url')) { if (t.hasOwnProperty('url')) {
var url = JSON.parse(t.url.toString()); var url = JSON.parse(t.url.toString());
logger.debug(url); logger.debug(url);
@ -779,14 +711,13 @@ router.post('/add', function(req, res) {
} }
res.writeHead(200, {ContentType: 'application/json'}); res.writeHead(200, {ContentType: 'application/json'});
res.end(JSON.stringify({adding: url})); res.end(JSON.stringify({adding: url}));
}); });
router.post('/redo', function(req, res) { router.post('/redo', function(req, res) {
logger.debug('redoing entry..'); logger.debug('redoing entry..');
var t = req.body; let t = req.body;
console.log(t); console.log(t);
if (t.hasOwnProperty('url')) { if (t.hasOwnProperty('url')) {
var url = t.url.toString(); var url = t.url.toString();
@ -797,22 +728,21 @@ router.post('/redo', function(req, res) {
} }
res.writeHead(200, {ContentType: 'application/json'}); res.writeHead(200, {ContentType: 'application/json'});
res.end(JSON.stringify({adding: url})); res.end(JSON.stringify({adding: url}));
}); });
router.route('/new') router.route('/new')
.get(function(req, res, next) { .get(function(req, res, next) {
logger.debug('Save new'); logger.debug('Save new');
busEmitter.emit('getBookmarkRes', req.query.url, res); busEmitter.emit('getBookmarkRes', req.query.url, res);
}).post(function(req, res, next) { }).post(function(req, res, next) {
logger.debug('Posted Save new'); logger.debug('Posted Save new');
logger.info(req.body); logger.info(req.body);
if (Object.keys(req.body).length !== 0) { if (Object.keys(req.body).length !== 0) {
busEmitter.emit('getBookmarkRes', req.body.url, res); busEmitter.emit('getBookmarkRes', req.body.url, res);
} else { } else {
res.status(422).end(); res.status(422).end();
} }
}); });
busEmitter.emit('updateTagsDB'); busEmitter.emit('updateTagsDB');

6
server/processor.js
View File

@ -19,3 +19,9 @@ var specialHandlers = [{
} }
]; ];
exports.processor = {
};