Doing some work on version 2
This commit is contained in:
martind2000
parent
8657f7362f
commit
b33aebfecc
155
.eslintrc
Normal file
155
.eslintrc
Normal file
@ -0,0 +1,155 @@
|
||||
{
|
||||
"parser": "babel-eslint",
|
||||
"rules": {
|
||||
/**
|
||||
* Strict mode
|
||||
*/
|
||||
// babel inserts "use strict"; for us
|
||||
"strict": [2, "never"], // http://eslint.org/docs/rules/strict
|
||||
|
||||
/**
|
||||
* ES6
|
||||
*/
|
||||
"no-var": 2, // http://eslint.org/docs/rules/no-var
|
||||
"prefer-const": 0, // http://eslint.org/docs/rules/prefer-const
|
||||
|
||||
/**
|
||||
* Variables
|
||||
*/
|
||||
"no-shadow": 2, // http://eslint.org/docs/rules/no-shadow
|
||||
"no-shadow-restricted-names": 2, // http://eslint.org/docs/rules/no-shadow-restricted-names
|
||||
"no-unused-vars": [2, { // http://eslint.org/docs/rules/no-unused-vars
|
||||
"vars": "local",
|
||||
"args": "none"
|
||||
}],
|
||||
"no-use-before-define": 0, // http://eslint.org/docs/rules/no-use-before-define
|
||||
|
||||
/**
|
||||
* Possible errors
|
||||
*/
|
||||
"comma-dangle": [2, "never"], // http://eslint.org/docs/rules/comma-dangle
|
||||
"no-cond-assign": [2, "always"], // http://eslint.org/docs/rules/no-cond-assign
|
||||
"no-console": 1, // http://eslint.org/docs/rules/no-console
|
||||
"no-debugger": 1, // http://eslint.org/docs/rules/no-debugger
|
||||
"no-alert": 1, // http://eslint.org/docs/rules/no-alert
|
||||
"no-constant-condition": 1, // http://eslint.org/docs/rules/no-constant-condition
|
||||
"no-dupe-keys": 2, // http://eslint.org/docs/rules/no-dupe-keys
|
||||
"no-duplicate-case": 2, // http://eslint.org/docs/rules/no-duplicate-case
|
||||
"no-empty": 2, // http://eslint.org/docs/rules/no-empty
|
||||
"no-ex-assign": 2, // http://eslint.org/docs/rules/no-ex-assign
|
||||
"no-extra-boolean-cast": 0, // http://eslint.org/docs/rules/no-extra-boolean-cast
|
||||
"no-extra-semi": 2, // http://eslint.org/docs/rules/no-extra-semi
|
||||
"no-func-assign": 2, // http://eslint.org/docs/rules/no-func-assign
|
||||
"no-inner-declarations": 2, // http://eslint.org/docs/rules/no-inner-declarations
|
||||
"no-invalid-regexp": 2, // http://eslint.org/docs/rules/no-invalid-regexp
|
||||
"no-irregular-whitespace": 2, // http://eslint.org/docs/rules/no-irregular-whitespace
|
||||
"no-obj-calls": 2, // http://eslint.org/docs/rules/no-obj-calls
|
||||
"no-sparse-arrays": 2, // http://eslint.org/docs/rules/no-sparse-arrays
|
||||
"no-unreachable": 2, // http://eslint.org/docs/rules/no-unreachable
|
||||
"use-isnan": 2, // http://eslint.org/docs/rules/use-isnan
|
||||
"block-scoped-var": 0, // http://eslint.org/docs/rules/block-scoped-var
|
||||
|
||||
/**
|
||||
* Best practices
|
||||
*/
|
||||
"consistent-return": 0, // http://eslint.org/docs/rules/consistent-return
|
||||
"curly": [2, "multi-line"], // http://eslint.org/docs/rules/curly
|
||||
"default-case": 2, // http://eslint.org/docs/rules/default-case
|
||||
"dot-notation": [2, { // http://eslint.org/docs/rules/dot-notation
|
||||
"allowKeywords": true
|
||||
}],
|
||||
"eqeqeq": 2, // http://eslint.org/docs/rules/eqeqeq
|
||||
"guard-for-in": 0, // http://eslint.org/docs/rules/guard-for-in
|
||||
"no-caller": 2, // http://eslint.org/docs/rules/no-caller
|
||||
"no-else-return": 2, // http://eslint.org/docs/rules/no-else-return
|
||||
"no-eq-null": 2, // http://eslint.org/docs/rules/no-eq-null
|
||||
"no-eval": 2, // http://eslint.org/docs/rules/no-eval
|
||||
"no-extend-native": 2, // http://eslint.org/docs/rules/no-extend-native
|
||||
"no-extra-bind": 2, // http://eslint.org/docs/rules/no-extra-bind
|
||||
"no-fallthrough": 2, // http://eslint.org/docs/rules/no-fallthrough
|
||||
"no-floating-decimal": 2, // http://eslint.org/docs/rules/no-floating-decimal
|
||||
"no-implied-eval": 2, // http://eslint.org/docs/rules/no-implied-eval
|
||||
"no-lone-blocks": 2, // http://eslint.org/docs/rules/no-lone-blocks
|
||||
"no-loop-func": 2, // http://eslint.org/docs/rules/no-loop-func
|
||||
"no-multi-str": 2, // http://eslint.org/docs/rules/no-multi-str
|
||||
"no-native-reassign": 2, // http://eslint.org/docs/rules/no-native-reassign
|
||||
"no-new": 2, // http://eslint.org/docs/rules/no-new
|
||||
"no-new-func": 2, // http://eslint.org/docs/rules/no-new-func
|
||||
"no-new-wrappers": 2, // http://eslint.org/docs/rules/no-new-wrappers
|
||||
"no-octal": 2, // http://eslint.org/docs/rules/no-octal
|
||||
"no-octal-escape": 2, // http://eslint.org/docs/rules/no-octal-escape
|
||||
"no-param-reassign": 0, // http://eslint.org/docs/rules/no-param-reassign
|
||||
"no-proto": 2, // http://eslint.org/docs/rules/no-proto
|
||||
"no-redeclare": 2, // http://eslint.org/docs/rules/no-redeclare
|
||||
// "no-return-assign": 2, // http://eslint.org/docs/rules/no-return-assign
|
||||
"no-script-url": 2, // http://eslint.org/docs/rules/no-script-url
|
||||
"no-self-compare": 2, // http://eslint.org/docs/rules/no-self-compare
|
||||
"no-sequences": 2, // http://eslint.org/docs/rules/no-sequences
|
||||
"no-throw-literal": 2, // http://eslint.org/docs/rules/no-throw-literal
|
||||
"no-with": 2, // http://eslint.org/docs/rules/no-with
|
||||
"radix": 2, // http://eslint.org/docs/rules/radix
|
||||
"vars-on-top": 2, // http://eslint.org/docs/rules/vars-on-top
|
||||
"wrap-iife": [2, "any"], // http://eslint.org/docs/rules/wrap-iife
|
||||
"yoda": 2, // http://eslint.org/docs/rules/yoda
|
||||
|
||||
/**
|
||||
* Style
|
||||
*/
|
||||
"indent": [2, 2], // http://eslint.org/docs/rules/indent
|
||||
"brace-style": [2, // http://eslint.org/docs/rules/brace-style
|
||||
"1tbs", {
|
||||
"allowSingleLine": true
|
||||
}],
|
||||
"quotes": [
|
||||
2, "single", "avoid-escape" // http://eslint.org/docs/rules/quotes
|
||||
],
|
||||
"camelcase": [2, { // http://eslint.org/docs/rules/camelcase
|
||||
"properties": "never"
|
||||
}],
|
||||
"comma-spacing": [2, { // http://eslint.org/docs/rules/comma-spacing
|
||||
"before": false,
|
||||
"after": true
|
||||
}],
|
||||
"comma-style": [2, "last"], // http://eslint.org/docs/rules/comma-style
|
||||
"eol-last": 2, // http://eslint.org/docs/rules/eol-last
|
||||
"func-names": 0, // http://eslint.org/docs/rules/func-names
|
||||
"key-spacing": [2, { // http://eslint.org/docs/rules/key-spacing
|
||||
"beforeColon": false,
|
||||
"afterColon": true
|
||||
}],
|
||||
"new-cap": [2, { // http://eslint.org/docs/rules/new-cap
|
||||
"newIsCap": true
|
||||
}],
|
||||
"no-multiple-empty-lines": [2, { // http://eslint.org/docs/rules/no-multiple-empty-lines
|
||||
"max": 2
|
||||
}],
|
||||
"no-nested-ternary": 2, // http://eslint.org/docs/rules/no-nested-ternary
|
||||
"no-new-object": 2, // http://eslint.org/docs/rules/no-new-object
|
||||
"no-spaced-func": 2, // http://eslint.org/docs/rules/no-spaced-func
|
||||
"no-trailing-spaces": 2, // http://eslint.org/docs/rules/no-trailing-spaces
|
||||
"no-extra-parens": [2, "functions"], // http://eslint.org/docs/rules/no-extra-parens
|
||||
"no-underscore-dangle": 0, // http://eslint.org/docs/rules/no-underscore-dangle
|
||||
"one-var": [2, "never"], // http://eslint.org/docs/rules/one-var
|
||||
"padded-blocks": [2, "never"], // http://eslint.org/docs/rules/padded-blocks
|
||||
"semi": [2, "always"], // http://eslint.org/docs/rules/semi
|
||||
"semi-spacing": [2, { // http://eslint.org/docs/rules/semi-spacing
|
||||
"before": false,
|
||||
"after": true
|
||||
}],
|
||||
"keyword-spacing": [2, { // http://eslint.org/docs/rules/keyword-spacing
|
||||
"before": true,
|
||||
"after": true
|
||||
}],
|
||||
"space-before-blocks": 2, // http://eslint.org/docs/rules/space-before-blocks
|
||||
"space-before-function-paren": [2, "never"], // http://eslint.org/docs/rules/space-before-function-paren
|
||||
"space-infix-ops": 2, // http://eslint.org/docs/rules/space-infix-ops
|
||||
"spaced-comment": [0, "always", { // http://eslint.org/docs/rules/spaced-comment
|
||||
"exceptions": ["*"],
|
||||
"markers": ["*"]
|
||||
}]
|
||||
},
|
||||
"env": {
|
||||
"node": true,
|
||||
"browser": true
|
||||
}
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
var express = require('express'), path = require('path'), http = require('http'),
|
||||
const express = require('express'), path = require('path'), http = require('http'),
|
||||
|
||||
|
||||
favicon = require('serve-favicon'),
|
||||
@ -8,7 +8,7 @@ var express = require('express'), path = require('path'), http = require('http')
|
||||
keeper = require('./server/keeperV2')
|
||||
|
||||
;
|
||||
var app = express();
|
||||
const app = express();
|
||||
|
||||
app.set('port', process.env.PORT || 8026);
|
||||
app.engine('html', require('ejs').renderFile);
|
||||
@ -32,6 +32,7 @@ app.use('/', keeper);
|
||||
|
||||
app.listen(app.get('port'), function() {
|
||||
console.log('Keeper Server listening on ' + app.get('port'));
|
||||
});/**
|
||||
});
|
||||
/**
|
||||
* Created by Martin on 22/02/2016.
|
||||
*/
|
||||
|
||||
@ -24,7 +24,9 @@
|
||||
"url": "^0.11.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"babel-eslint": "^7.2.3",
|
||||
"del": "^2.2.0",
|
||||
"eslint": "^4.1.1",
|
||||
"gulp": "^3.9.1",
|
||||
"gulp-autoprefixer": "^3.1.0",
|
||||
"gulp-cache": "^0.4.2",
|
||||
|
||||
28
server/dbcloudant.js
Normal file
28
server/dbcloudant.js
Normal file
@ -0,0 +1,28 @@
|
||||
/**
|
||||
* Created by mdonnel on 27/06/2017.
|
||||
*/
|
||||
|
||||
/*
|
||||
|
||||
We've moved to cloudant through IBM Bluemix for the database
|
||||
|
||||
https://25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix.cloudant.com/dashboard.html#usage
|
||||
|
||||
|
||||
*/
|
||||
|
||||
const credentials = {
|
||||
'username': '25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix',
|
||||
'password': '8e417af1b0462ca55726848846cc6b8696fc76defe9d1864cbc334be59549e0c',
|
||||
'host': '25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix.cloudant.com',
|
||||
'port': 443,
|
||||
'url': 'https://25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix:8e417af1b0462ca55726848846cc6b8696fc76defe9d1864cbc334be59549e0c@25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix.cloudant.com',
|
||||
'database': 'keeper'
|
||||
};
|
||||
|
||||
const cloudant = require('cloudant');
|
||||
const cloudantConnection = cloudant({account: credentials.username, password: credentials.password});
|
||||
|
||||
exports.dbcloudant = cloudantConnection.db.use(credentials.database);
|
||||
|
||||
|
||||
@ -1,56 +1,36 @@
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Created by Martin on 22/02/2016.
|
||||
*/
|
||||
var express = require('express');
|
||||
var http = require('http'), request = require('request'), cheerio = require(
|
||||
'cheerio'), util = require('util');
|
||||
var jsonfile = require('jsonfile'), fs = require('fs'), STRING = require(
|
||||
'string');
|
||||
var converter = require('html-to-markdown');
|
||||
var zlib = require('zlib');
|
||||
var log4js = require('log4js');
|
||||
var logger = log4js.getLogger();
|
||||
var URL = require('url');
|
||||
let express = require('express');
|
||||
let http = require('http'), request = require('request'), cheerio = require('cheerio'), util = require('util');
|
||||
let jsonfile = require('jsonfile'), fs = require('fs'), STRING = require('string');
|
||||
let converter = require('html-to-markdown');
|
||||
let zlib = require('zlib');
|
||||
let log4js = require('log4js');
|
||||
let logger = log4js.getLogger();
|
||||
let URL = require('url');
|
||||
|
||||
var router = express.Router();
|
||||
let router = express.Router();
|
||||
|
||||
var EventEmitter = require('events');
|
||||
let EventEmitter = require('events');
|
||||
|
||||
//var nano = require('nano')('http://martind2000:1V3D4m526i@localhost:5984');
|
||||
var busEmitter = new EventEmitter();
|
||||
let busEmitter = new EventEmitter();
|
||||
|
||||
const dbCloudant = require('./dbcloudant').dbcloudant;
|
||||
|
||||
var db_name = 'keeper';
|
||||
//var dbCouch = nano.use(db_name);
|
||||
|
||||
/*
|
||||
|
||||
We've moved to cloudant through IBM Bluemix for the database
|
||||
|
||||
https://25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix.cloudant.com/dashboard.html#usage
|
||||
|
||||
|
||||
*/
|
||||
|
||||
var credentials = {
|
||||
"username": "25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix",
|
||||
"password": "8e417af1b0462ca55726848846cc6b8696fc76defe9d1864cbc334be59549e0c",
|
||||
"host": "25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix.cloudant.com",
|
||||
"port": 443,
|
||||
"url": "https://25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix:8e417af1b0462ca55726848846cc6b8696fc76defe9d1864cbc334be59549e0c@25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix.cloudant.com",
|
||||
"database" : "keeper"
|
||||
};
|
||||
|
||||
var Cloudant = require('cloudant');
|
||||
var cloudant = Cloudant({account:credentials.username, password:credentials.password});
|
||||
|
||||
var dbCloudant = cloudant.db.use(credentials.database);
|
||||
|
||||
|
||||
var jsonFile = __dirname + '/' + 'output.json';
|
||||
var bodyfile = __dirname + '/' + 'body.html';
|
||||
var htmlfile = __dirname + '/' + 'testoutput.html';
|
||||
var generics = [
|
||||
|
||||
let jsonFile = __dirname + '/' + 'output.json';
|
||||
let bodyfile = __dirname + '/' + 'body.html';
|
||||
let htmlfile = __dirname + '/' + 'testoutput.html';
|
||||
let generics = [
|
||||
'ARTICLE',
|
||||
'div.content_column',
|
||||
'div.post',
|
||||
@ -59,7 +39,7 @@ var generics = [
|
||||
'div.content.body'
|
||||
];
|
||||
|
||||
var specialHandlers = [{
|
||||
let specialHandlers = [{
|
||||
url: 'www.reddit.com', fn: function(body, url) {
|
||||
return doReddit(body, url);
|
||||
}
|
||||
@ -76,16 +56,13 @@ var specialHandlers = [{
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
];
|
||||
|
||||
|
||||
|
||||
function cleaner(b) {
|
||||
var _b = b;
|
||||
let _b = b;
|
||||
|
||||
var unwanted = [
|
||||
let unwanted = [
|
||||
'LINK',
|
||||
'META',
|
||||
'TITLE',
|
||||
@ -117,7 +94,7 @@ function cleaner(b) {
|
||||
|
||||
];
|
||||
|
||||
for (var i = 0; i < unwanted.length; i++) {
|
||||
for (let i = 0; i < unwanted.length; i++) {
|
||||
_b.find(unwanted[i]).remove();
|
||||
}
|
||||
|
||||
@ -139,7 +116,7 @@ function insertBookmark(obj) {
|
||||
|
||||
function updateBookmark(obj, _id, _rev) {
|
||||
logger.debug('Updating couch...');
|
||||
var _obj = obj;
|
||||
let _obj = obj;
|
||||
_obj._id = _id;
|
||||
_obj._rev = _rev;
|
||||
|
||||
@ -148,49 +125,46 @@ function updateBookmark(obj, _id, _rev) {
|
||||
if (err) {
|
||||
logger.error('Error updating into couch');
|
||||
return;
|
||||
} else {
|
||||
}
|
||||
logger.info('I think we updated ok...');
|
||||
busEmitter.emit('updateTagsDB');
|
||||
|
||||
}
|
||||
});
|
||||
logger.debug('Update done..');
|
||||
}
|
||||
var doInsertBookmark = (obj) => {
|
||||
let doInsertBookmark = (obj) => {
|
||||
// Logger.info('sendSocket: ' + JSON.stringify(obj));
|
||||
insertBookmark(obj);
|
||||
};
|
||||
|
||||
var doUpdateBookmark = (obj, _id, _rev) => {
|
||||
let doUpdateBookmark = (obj, _id, _rev) => {
|
||||
// Logger.info('sendSocket: ' + JSON.stringify(obj));
|
||||
updateBookmark(obj, _id, _rev);
|
||||
};
|
||||
|
||||
var doGetBookmark = (obj) => {
|
||||
let doGetBookmark = (obj) => {
|
||||
// Logger.info('sendSocket: ' + JSON.stringify(obj));
|
||||
genericGrab(obj);
|
||||
};
|
||||
|
||||
var doGetBookmarkRedo = (obj) => {
|
||||
let doGetBookmarkRedo = (obj) => {
|
||||
// Logger.info('sendSocket: ' + JSON.stringify(obj));
|
||||
genericGrab(obj);
|
||||
};
|
||||
|
||||
var doGetBookmarkRes = (url, res) => {
|
||||
let doGetBookmarkRes = (url, res) => {
|
||||
logger.debug('doGetBookmarkRes');
|
||||
// Logger.info('sendSocket: ' + JSON.stringify(obj));
|
||||
genericGrab(url, res);
|
||||
};
|
||||
|
||||
var doUpdateTagsDB = () => {
|
||||
let doUpdateTagsDB = () => {
|
||||
logger.debug('Update the tags database...');
|
||||
|
||||
// dbCouch.view('getAllTags', 'getAllTags', function(err, body) {
|
||||
dbCloudant.view('getAllTags', 'getAllTags', function(err, body) {
|
||||
var masterList = [];
|
||||
let masterList = [];
|
||||
if (!err) {
|
||||
body.rows.forEach(function(doc) {
|
||||
|
||||
masterList = masterList.concat(doc.value);
|
||||
});
|
||||
|
||||
@ -202,29 +176,25 @@ var doUpdateTagsDB = () => {
|
||||
dbCloudant.view('taglist', 'taglist', function(err, body) {
|
||||
// Logger.debug(body);
|
||||
if (!err) {
|
||||
|
||||
var outJSON = {};
|
||||
let outJSON = {};
|
||||
|
||||
body.rows.forEach(function(doc) {
|
||||
doSaveTagsDB(doc.value, masterList);
|
||||
});
|
||||
|
||||
} else {
|
||||
logger.error('NO TAG LIST EXISTS');
|
||||
}
|
||||
});
|
||||
|
||||
} else {
|
||||
|
||||
}
|
||||
});
|
||||
|
||||
};
|
||||
|
||||
var doSaveTagsDB = (orig, newList) => {
|
||||
logger.debug('doSaveTagsDB');
|
||||
|
||||
var _obj = orig;
|
||||
let _obj = orig;
|
||||
|
||||
_obj.taglist = newList;
|
||||
|
||||
@ -233,10 +203,8 @@ var doSaveTagsDB = (orig, newList) => {
|
||||
if (err) {
|
||||
logger.error('Error updating into couch');
|
||||
return;
|
||||
} else {
|
||||
logger.info('Updated the tags list...');
|
||||
|
||||
}
|
||||
logger.info('Updated the tags list...');
|
||||
});
|
||||
};
|
||||
|
||||
@ -252,10 +220,10 @@ busEmitter.on('saveTagsDB', doSaveTagsDB);
|
||||
|
||||
function doEngadget(body, url) {
|
||||
logger.info('GRABBING Engadget');
|
||||
var obj = {}, tdihbody, i, urlObj, urlPrefix;
|
||||
let obj = {}, tdihbody, i, urlObj, urlPrefix;
|
||||
|
||||
var $ = cheerio.load(body);
|
||||
var title = $('TITLE').text();
|
||||
let $ = cheerio.load(body);
|
||||
let title = $('TITLE').text();
|
||||
|
||||
tdihbody = $('DIV#page_body');
|
||||
|
||||
@ -284,10 +252,8 @@ function doEngadget(body, url) {
|
||||
|
||||
$(this).attr('src', s);
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
catch (e) {
|
||||
} catch (e) {
|
||||
logger.error(e);
|
||||
}
|
||||
|
||||
@ -304,10 +270,10 @@ function doEngadget(body, url) {
|
||||
|
||||
function doAndroidDeveloper(body, url) {
|
||||
logger.info('GRABBING AndroidDeveloper');
|
||||
var obj = {}, tdihbody, i, urlObj, urlPrefix;
|
||||
let obj = {}, tdihbody, i, urlObj, urlPrefix;
|
||||
|
||||
var $ = cheerio.load(body);
|
||||
var title = $('TITLE').text();
|
||||
let $ = cheerio.load(body);
|
||||
let title = $('TITLE').text();
|
||||
|
||||
tdihbody = $('DIV.jd-descr');
|
||||
|
||||
@ -336,10 +302,8 @@ function doAndroidDeveloper(body, url) {
|
||||
|
||||
$(this).attr('src', s);
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
catch (e) {
|
||||
} catch (e) {
|
||||
logger.error(e);
|
||||
}
|
||||
|
||||
@ -356,15 +320,14 @@ function doAndroidDeveloper(body, url) {
|
||||
|
||||
function doReddit(body, url) {
|
||||
logger.info('GRABBING REDDIT');
|
||||
var obj = {}, tdihbody, i, urlObj, urlPrefix;
|
||||
let obj = {}, tdihbody, i, urlObj, urlPrefix;
|
||||
|
||||
var $ = cheerio.load(body);
|
||||
var title = $('TITLE').text();
|
||||
let $ = cheerio.load(body);
|
||||
let title = $('TITLE').text();
|
||||
|
||||
tdihbody = $('DIV.entry');
|
||||
|
||||
tdihbody.find('A.thumbnail').each(function(i, elem) {
|
||||
|
||||
logger.warn($(this));
|
||||
});
|
||||
|
||||
@ -386,12 +349,11 @@ function doReddit(body, url) {
|
||||
}
|
||||
|
||||
|
||||
|
||||
function genericProcessor(body, url) {
|
||||
logger.info('USING DEFAULT PROCESSOR');
|
||||
var obj = {}, tdihbody, i, urlObj, urlPrefix;
|
||||
var $ = cheerio.load(body);
|
||||
var title = $('TITLE').text();
|
||||
let obj = {}, tdihbody, i, urlObj, urlPrefix;
|
||||
let $ = cheerio.load(body);
|
||||
let title = $('TITLE').text();
|
||||
|
||||
i = 0;
|
||||
|
||||
@ -406,16 +368,13 @@ function genericProcessor(body, url) {
|
||||
logger.debug(tdihbody.length);
|
||||
tdihbody = cleaner(tdihbody);
|
||||
logger.debug(title);
|
||||
|
||||
} else {
|
||||
logger.warn('Using whole body');
|
||||
// Bah. nothing to reduce so just grab the body, tidy it and use that
|
||||
tdihbody = $('BODY');
|
||||
|
||||
if (tdihbody.length === 0) {
|
||||
|
||||
tdihbody = $(':root');
|
||||
|
||||
}
|
||||
|
||||
logger.debug(tdihbody.length);
|
||||
@ -447,10 +406,8 @@ function genericProcessor(body, url) {
|
||||
|
||||
$(this).attr('src', s);
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
catch (e) {
|
||||
} catch (e) {
|
||||
logger.error(e);
|
||||
}
|
||||
|
||||
@ -465,10 +422,7 @@ function genericProcessor(body, url) {
|
||||
return obj;
|
||||
}
|
||||
function processBody(body, url, _id, _rev) {
|
||||
|
||||
var obj = {}, i, urlObj, urlPrefix;
|
||||
|
||||
|
||||
let obj = {}, i, urlObj, urlPrefix;
|
||||
|
||||
|
||||
// Try to find a body to grab
|
||||
@ -477,7 +431,7 @@ function processBody(body, url, _id, _rev) {
|
||||
|
||||
logger.debug('host:', urlObj.host);
|
||||
|
||||
var flag;
|
||||
let flag;
|
||||
for (i = 0; i < specialHandlers.length; i++) {
|
||||
if (urlObj.host === specialHandlers[i].url) {
|
||||
flag = true;
|
||||
@ -506,11 +460,9 @@ function processBody(body, url, _id, _rev) {
|
||||
}
|
||||
|
||||
return obj;
|
||||
|
||||
}
|
||||
function genericGrab(obj, res) {
|
||||
|
||||
var url, _id = null, _ver = null;
|
||||
let url, _id = null, _ver = null;
|
||||
|
||||
if (typeof obj === 'string') {
|
||||
logger.info(obj);
|
||||
@ -527,7 +479,7 @@ function genericGrab(obj, res) {
|
||||
logger.info(_id);
|
||||
logger.info(_ver);
|
||||
|
||||
var options = {
|
||||
let options = {
|
||||
url: url,
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36'
|
||||
@ -538,17 +490,15 @@ function genericGrab(obj, res) {
|
||||
};
|
||||
|
||||
request(options, function(err, resp, body) {
|
||||
if (err)
|
||||
throw err;
|
||||
if (err) {throw err;}
|
||||
|
||||
if (resp.headers.hasOwnProperty('content-encoding')) {
|
||||
logger.warn('content-encoding');
|
||||
if (resp.headers['content-encoding'] == 'gzip') {
|
||||
|
||||
// to test http://chaosinthekitchen.com/2009/07/lime-and-coconut-chicken/
|
||||
|
||||
var gunzip = zlib.createGunzip();
|
||||
var jsonString = '';
|
||||
let gunzip = zlib.createGunzip();
|
||||
let jsonString = '';
|
||||
resp.pipe(gunzip);
|
||||
gunzip.on('data', function(chunk) {
|
||||
jsonString += chunk;
|
||||
@ -566,14 +516,12 @@ function genericGrab(obj, res) {
|
||||
res.render('grabbed');
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
var b = processBody(body, url, _id, _ver);
|
||||
if (res != null) {
|
||||
res.render('grabbed', {data: b});
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
@ -584,21 +532,18 @@ router.get('/pocket', function(req, res) {
|
||||
// dbCouch.view('pocketList', 'pocketList', function(err, body) {
|
||||
dbCloudant.view('pocketList', 'pocketList', function(err, body) {
|
||||
if (!err) {
|
||||
|
||||
var outJSON = [];
|
||||
let outJSON = [];
|
||||
body.rows.forEach(function(doc) {
|
||||
var obj = {id: doc.id, entry: doc.value};
|
||||
let obj = {id: doc.id, entry: doc.value};
|
||||
console.log(typeof obj.entry.tn);
|
||||
if (typeof obj.entry.tn === 'string') {
|
||||
console.log('its a string:', typeof obj.entry.tn)
|
||||
console.log('its a string:', typeof obj.entry.tn);
|
||||
obj.entry.tn = 'http://image.silvrtree.co.uk/100,fit,q80/' + obj.entry.tn;
|
||||
} else {
|
||||
obj.entry.tn = 'gfx/fm.png';
|
||||
}
|
||||
|
||||
outJSON.push(obj);
|
||||
|
||||
|
||||
});
|
||||
|
||||
logger.debug(util.inspect(body));
|
||||
@ -612,15 +557,13 @@ router.get('/pocket', function(req, res) {
|
||||
});
|
||||
|
||||
|
||||
|
||||
router.get('/list', function(req, res) {
|
||||
logger.debug('list..');
|
||||
|
||||
//dbCouch.view('titles', 'titles', function(err, body) {
|
||||
dbCloudant.view('titles', 'titles', function(err, body) {
|
||||
if (!err) {
|
||||
|
||||
var outJSON = [];
|
||||
let outJSON = [];
|
||||
body.rows.forEach(function(doc) {
|
||||
outJSON.push({id: doc.id, title: doc.value});
|
||||
});
|
||||
@ -628,7 +571,6 @@ router.get('/list', function(req, res) {
|
||||
//Logger.debug(util.inspect(body));
|
||||
res.writeHead(200, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({list: outJSON}));
|
||||
|
||||
} else {
|
||||
res.writeHead(500, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({}));
|
||||
@ -644,8 +586,7 @@ router.get('/entry/:id', function(req, res) {
|
||||
//dbCouch.get(req.params.id, function(err, body) {
|
||||
dbCloudant.get(req.params.id, function(err, body) {
|
||||
if (!err) {
|
||||
|
||||
var outJSON = {};
|
||||
let outJSON = {};
|
||||
outJSON._id = body._id;
|
||||
outJSON._rev = body._rev;
|
||||
outJSON.title = body.title;
|
||||
@ -656,13 +597,11 @@ router.get('/entry/:id', function(req, res) {
|
||||
//Logger.debug(util.inspect(body));
|
||||
res.writeHead(200, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify(outJSON));
|
||||
|
||||
} else {
|
||||
res.writeHead(500, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({}));
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
router.route('/tags')
|
||||
@ -675,7 +614,7 @@ router.route('/tags')
|
||||
dbCloudant.view('taglist', 'taglist', function(err, body) {
|
||||
if (!err) {
|
||||
logger.debug(body);
|
||||
var outJSON = [];
|
||||
let outJSON = [];
|
||||
body.rows.forEach(function(doc) {
|
||||
logger.info(doc.value.taglist);
|
||||
if (doc.value[0] == req.params.id) {
|
||||
@ -686,16 +625,14 @@ router.route('/tags')
|
||||
//Logger.debug(util.inspect(body));
|
||||
res.writeHead(200, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({list: outJSON}));
|
||||
|
||||
} else {
|
||||
logger.error(err);
|
||||
res.writeHead(500, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({}));
|
||||
}
|
||||
});
|
||||
|
||||
}).post(function(req, res, next) {
|
||||
var t = req.body;
|
||||
let t = req.body;
|
||||
console.log(t);
|
||||
|
||||
logger.info('regetting:' + req.body._id);
|
||||
@ -703,8 +640,7 @@ router.route('/tags')
|
||||
//dbCouch.get(req.body._id, function(err, body) {
|
||||
dbCloudant.get(req.body._id, function(err, body) {
|
||||
if (!err) {
|
||||
|
||||
var obj = {};
|
||||
let obj = {};
|
||||
|
||||
obj.url = body.url;
|
||||
obj.html = body.html;
|
||||
@ -715,7 +651,7 @@ router.route('/tags')
|
||||
logger.info('Updating...');
|
||||
busEmitter.emit('updateBookmarkData', obj, body._id, body._rev, res);
|
||||
|
||||
var outJSON = {};
|
||||
let outJSON = {};
|
||||
outJSON._id = body._id;
|
||||
outJSON._rev = body._rev;
|
||||
outJSON.title = body.title;
|
||||
@ -726,13 +662,11 @@ router.route('/tags')
|
||||
//Logger.debug(util.inspect(body));
|
||||
res.writeHead(200, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify(outJSON));
|
||||
|
||||
} else {
|
||||
res.writeHead(500, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({}));
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
|
||||
@ -745,31 +679,29 @@ router.get('/tags/:id', function(req, res) {
|
||||
dbCloudant.view('getTagByKey', 'getTagByKey', function(err, body) {
|
||||
if (!err) {
|
||||
// Logger.debug(body);
|
||||
var outJSON = [];
|
||||
let outJSON = [];
|
||||
body.rows.forEach(function(doc) {
|
||||
// Logger.debug(doc);
|
||||
if (doc.value[0] == req.params.id) {
|
||||
outJSON.push({id: doc.id, title: doc.value[1]})
|
||||
outJSON.push({id: doc.id, title: doc.value[1]});
|
||||
}
|
||||
});
|
||||
|
||||
//Logger.debug(util.inspect(body));
|
||||
res.writeHead(200, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({list: outJSON}));
|
||||
|
||||
} else {
|
||||
logger.error(err);
|
||||
res.writeHead(500, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({}));
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
router.post('/add', function(req, res) {
|
||||
logger.debug('add entry..');
|
||||
|
||||
var t = req.body;
|
||||
let t = req.body;
|
||||
if (t.hasOwnProperty('url')) {
|
||||
var url = JSON.parse(t.url.toString());
|
||||
logger.debug(url);
|
||||
@ -779,14 +711,13 @@ router.post('/add', function(req, res) {
|
||||
}
|
||||
res.writeHead(200, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({adding: url}));
|
||||
|
||||
});
|
||||
|
||||
|
||||
router.post('/redo', function(req, res) {
|
||||
logger.debug('redoing entry..');
|
||||
|
||||
var t = req.body;
|
||||
let t = req.body;
|
||||
console.log(t);
|
||||
if (t.hasOwnProperty('url')) {
|
||||
var url = t.url.toString();
|
||||
@ -797,7 +728,6 @@ router.post('/redo', function(req, res) {
|
||||
}
|
||||
res.writeHead(200, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({adding: url}));
|
||||
|
||||
});
|
||||
|
||||
router.route('/new')
|
||||
|
||||
@ -19,3 +19,9 @@ var specialHandlers = [{
|
||||
}
|
||||
|
||||
];
|
||||
|
||||
|
||||
exports.processor = {
|
||||
|
||||
};
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user