'update'
This commit is contained in:
parent
c4b79ea932
commit
8657f7362f
1
.gitignore
vendored
1
.gitignore
vendored
@ -77,3 +77,4 @@ build/Release
|
||||
# https://docs.npmjs.com/misc/faq#should-i-check-my-node-modules-folder-into-git
|
||||
node_modules
|
||||
|
||||
/jspm_packages/
|
||||
|
12
app/browserconfig.xml
Normal file
12
app/browserconfig.xml
Normal file
@ -0,0 +1,12 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<browserconfig>
|
||||
<msapplication>
|
||||
<tile>
|
||||
<square70x70logo src="/fav/mstile-70x70.png"/>
|
||||
<square150x150logo src="/fav/mstile-150x150.png"/>
|
||||
<square310x310logo src="/fav/mstile-310x310.png"/>
|
||||
<wide310x150logo src="/fav/mstile-310x150.png"/>
|
||||
<TileColor>#ffc40d</TileColor>
|
||||
</tile>
|
||||
</msapplication>
|
||||
</browserconfig>
|
@ -1,12 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<browserconfig>
|
||||
<msapplication>
|
||||
<tile>
|
||||
<square70x70logo src="/mstile-70x70.png"/>
|
||||
<square150x150logo src="/mstile-150x150.png"/>
|
||||
<square310x310logo src="/mstile-310x310.png"/>
|
||||
<wide310x150logo src="/mstile-310x150.png"/>
|
||||
<TileColor>#ffc40d</TileColor>
|
||||
</tile>
|
||||
</msapplication>
|
||||
</browserconfig>
|
14
app/indexV2.html
Normal file
14
app/indexV2.html
Normal file
@ -0,0 +1,14 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<title>Title</title>
|
||||
<script src="libs/underscore.js"></script>
|
||||
<script src="libs/backbone.js"></script>
|
||||
<script src="js/modules/lister.js"></script>
|
||||
<script src="js/appV2.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
</body>
|
||||
</html>
|
8
app/js/appV2.js
Normal file
8
app/js/appV2.js
Normal file
@ -0,0 +1,8 @@
|
||||
/**
|
||||
* Created by mdonnel on 19/04/2017.
|
||||
*/
|
||||
|
||||
(function(w) {
|
||||
|
||||
w.listModel = new ListerModel();
|
||||
})(window);
|
36
app/js/modules/lister.js
Normal file
36
app/js/modules/lister.js
Normal file
@ -0,0 +1,36 @@
|
||||
/**
|
||||
* Created by mdonnel on 19/04/2017.
|
||||
*/
|
||||
let ListerModel = Backbone.Model.extend({
|
||||
initialize: function() {
|
||||
|
||||
let url = '/list';
|
||||
this.set('url', url);
|
||||
this.update();
|
||||
},
|
||||
update: function() {
|
||||
this.getList();
|
||||
},
|
||||
getList: function() {
|
||||
let list = this.get('url');
|
||||
|
||||
$.ajax({
|
||||
type: 'GET', url: url, data: '', dataType: 'json',
|
||||
|
||||
timeout: 10000,
|
||||
|
||||
// ContentType: ('application/json'),
|
||||
headers: {
|
||||
'Access-Control-Allow-Origin': '*',
|
||||
'Access-Control-Allow-Methods': 'PUT, GET, POST, DELETE, OPTIONS',
|
||||
'Access-Control-Allow-Headers': 'Content-Type'
|
||||
|
||||
}, success: data => {
|
||||
// Console.log(data);
|
||||
this.set('list', data);
|
||||
}, error: (xhr, type) => {
|
||||
|
||||
}
|
||||
}).bind(this);
|
||||
}
|
||||
});
|
1920
app/libs/backbone.js
Normal file
1920
app/libs/backbone.js
Normal file
File diff suppressed because it is too large
Load Diff
1548
app/libs/underscore.js
Normal file
1548
app/libs/underscore.js
Normal file
File diff suppressed because it is too large
Load Diff
2
app/libs/zepto.min.js
vendored
Normal file
2
app/libs/zepto.min.js
vendored
Normal file
File diff suppressed because one or more lines are too long
@ -2,37 +2,37 @@
|
||||
"name": "Keeper",
|
||||
"icons": [
|
||||
{
|
||||
"src": "\/android-chrome-36x36.png",
|
||||
"src": "\/fav\/android-chrome-36x36.png",
|
||||
"sizes": "36x36",
|
||||
"type": "image\/png",
|
||||
"density": 0.75
|
||||
},
|
||||
{
|
||||
"src": "\/android-chrome-48x48.png",
|
||||
"src": "\/fav\/android-chrome-48x48.png",
|
||||
"sizes": "48x48",
|
||||
"type": "image\/png",
|
||||
"density": 1
|
||||
},
|
||||
{
|
||||
"src": "\/android-chrome-72x72.png",
|
||||
"src": "\/fav\/android-chrome-72x72.png",
|
||||
"sizes": "72x72",
|
||||
"type": "image\/png",
|
||||
"density": 1.5
|
||||
},
|
||||
{
|
||||
"src": "\/android-chrome-96x96.png",
|
||||
"src": "\/fav\/android-chrome-96x96.png",
|
||||
"sizes": "96x96",
|
||||
"type": "image\/png",
|
||||
"density": 2
|
||||
},
|
||||
{
|
||||
"src": "\/android-chrome-144x144.png",
|
||||
"src": "\/fav\/android-chrome-144x144.png",
|
||||
"sizes": "144x144",
|
||||
"type": "image\/png",
|
||||
"density": 3
|
||||
},
|
||||
{
|
||||
"src": "\/android-chrome-192x192.png",
|
||||
"src": "\/fav\/android-chrome-192x192.png",
|
||||
"sizes": "192x192",
|
||||
"type": "image\/png",
|
||||
"density": 4
|
76
config.js
Normal file
76
config.js
Normal file
@ -0,0 +1,76 @@
|
||||
System.config({
|
||||
baseURL: "/",
|
||||
defaultJSExtensions: true,
|
||||
transpiler: "babel",
|
||||
babelOptions: {
|
||||
"optional": [
|
||||
"runtime",
|
||||
"optimisation.modules.system"
|
||||
]
|
||||
},
|
||||
paths: {
|
||||
"github:*": "jspm_packages/github/*",
|
||||
"npm:*": "jspm_packages/npm/*"
|
||||
},
|
||||
|
||||
map: {
|
||||
"babel": "npm:babel-core@5.8.38",
|
||||
"babel-runtime": "npm:babel-runtime@5.8.38",
|
||||
"core-js": "npm:core-js@1.2.7",
|
||||
"github:jspm/nodelibs-assert@0.1.0": {
|
||||
"assert": "npm:assert@1.4.1"
|
||||
},
|
||||
"github:jspm/nodelibs-buffer@0.1.1": {
|
||||
"buffer": "npm:buffer@5.0.6"
|
||||
},
|
||||
"github:jspm/nodelibs-path@0.1.0": {
|
||||
"path-browserify": "npm:path-browserify@0.0.0"
|
||||
},
|
||||
"github:jspm/nodelibs-process@0.1.2": {
|
||||
"process": "npm:process@0.11.9"
|
||||
},
|
||||
"github:jspm/nodelibs-util@0.1.0": {
|
||||
"util": "npm:util@0.10.3"
|
||||
},
|
||||
"github:jspm/nodelibs-vm@0.1.0": {
|
||||
"vm-browserify": "npm:vm-browserify@0.0.4"
|
||||
},
|
||||
"npm:assert@1.4.1": {
|
||||
"assert": "github:jspm/nodelibs-assert@0.1.0",
|
||||
"buffer": "github:jspm/nodelibs-buffer@0.1.1",
|
||||
"process": "github:jspm/nodelibs-process@0.1.2",
|
||||
"util": "npm:util@0.10.3"
|
||||
},
|
||||
"npm:babel-runtime@5.8.38": {
|
||||
"process": "github:jspm/nodelibs-process@0.1.2"
|
||||
},
|
||||
"npm:buffer@5.0.6": {
|
||||
"base64-js": "npm:base64-js@1.2.0",
|
||||
"ieee754": "npm:ieee754@1.1.8"
|
||||
},
|
||||
"npm:core-js@1.2.7": {
|
||||
"fs": "github:jspm/nodelibs-fs@0.1.2",
|
||||
"path": "github:jspm/nodelibs-path@0.1.0",
|
||||
"process": "github:jspm/nodelibs-process@0.1.2",
|
||||
"systemjs-json": "github:systemjs/plugin-json@0.1.2"
|
||||
},
|
||||
"npm:inherits@2.0.1": {
|
||||
"util": "github:jspm/nodelibs-util@0.1.0"
|
||||
},
|
||||
"npm:path-browserify@0.0.0": {
|
||||
"process": "github:jspm/nodelibs-process@0.1.2"
|
||||
},
|
||||
"npm:process@0.11.9": {
|
||||
"assert": "github:jspm/nodelibs-assert@0.1.0",
|
||||
"fs": "github:jspm/nodelibs-fs@0.1.2",
|
||||
"vm": "github:jspm/nodelibs-vm@0.1.0"
|
||||
},
|
||||
"npm:util@0.10.3": {
|
||||
"inherits": "npm:inherits@2.0.1",
|
||||
"process": "github:jspm/nodelibs-process@0.1.2"
|
||||
},
|
||||
"npm:vm-browserify@0.0.4": {
|
||||
"indexof": "npm:indexof@0.0.1"
|
||||
}
|
||||
}
|
||||
});
|
@ -11,16 +11,25 @@ var express = require('express'), path = require('path'), http = require('http')
|
||||
var app = express();
|
||||
|
||||
app.set('port', process.env.PORT || 8026);
|
||||
app.set('views', path.join(__dirname, 'views'));
|
||||
app.set('view engine', 'ejs');
|
||||
app.engine('html', require('ejs').renderFile);
|
||||
//app.set('views', path.join(__dirname, 'views'));
|
||||
//app.set('view engine', 'ejs');
|
||||
app.set('view engine', 'html');
|
||||
|
||||
app.use(logger('dev'));
|
||||
app.use(bodyParser.json());
|
||||
app.use(bodyParser.urlencoded({ extended: true }));
|
||||
app.use(cookieParser());
|
||||
app.use(express.static(path.join(__dirname, 'dist')));
|
||||
|
||||
//app.use(express.static(path.join(__dirname, 'dist')));
|
||||
app.use(express.static(path.join(__dirname, 'app')));
|
||||
|
||||
app.use('/', keeper);
|
||||
|
||||
app.get('/v2', function(req, res) {
|
||||
res.render('../app/indexV2.html');
|
||||
});
|
||||
|
||||
/**
|
||||
* create the server
|
||||
*/
|
||||
|
37
keeper-serverV2.js
Normal file
37
keeper-serverV2.js
Normal file
@ -0,0 +1,37 @@
|
||||
var express = require('express'), path = require('path'), http = require('http'),
|
||||
|
||||
|
||||
favicon = require('serve-favicon'),
|
||||
logger = require('morgan'),
|
||||
cookieParser = require('cookie-parser'),
|
||||
bodyParser = require('body-parser'),
|
||||
keeper = require('./server/keeperV2')
|
||||
|
||||
;
|
||||
var app = express();
|
||||
|
||||
app.set('port', process.env.PORT || 8026);
|
||||
app.engine('html', require('ejs').renderFile);
|
||||
//app.set('views', path.join(__dirname, 'views'));
|
||||
//app.set('view engine', 'ejs');
|
||||
app.set('view engine', 'html');
|
||||
|
||||
app.use(logger('dev'));
|
||||
app.use(bodyParser.json());
|
||||
app.use(bodyParser.urlencoded({ extended: true }));
|
||||
app.use(cookieParser());
|
||||
|
||||
//app.use(express.static(path.join(__dirname, 'dist')));
|
||||
app.use(express.static(path.join(__dirname, 'app')));
|
||||
|
||||
app.use('/', keeper);
|
||||
|
||||
/**
|
||||
* create the server
|
||||
*/
|
||||
|
||||
app.listen(app.get('port'), function () {
|
||||
console.log('Keeper Server listening on ' + app.get('port'));
|
||||
});/**
|
||||
* Created by Martin on 22/02/2016.
|
||||
*/
|
@ -43,5 +43,13 @@
|
||||
"jsonfile": "^2.2.3",
|
||||
"log4js": "^0.6.31",
|
||||
"simplecrawler": "^0.6.2"
|
||||
},
|
||||
"jspm": {
|
||||
"dependencies": {},
|
||||
"devDependencies": {
|
||||
"babel": "npm:babel-core@^5.8.24",
|
||||
"babel-runtime": "npm:babel-runtime@^5.8.24",
|
||||
"core-js": "npm:core-js@^1.1.4"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
819
server/keeperV2.js
Normal file
819
server/keeperV2.js
Normal file
@ -0,0 +1,819 @@
|
||||
'use strict';
|
||||
/**
|
||||
* Created by Martin on 22/02/2016.
|
||||
*/
|
||||
var express = require('express');
|
||||
var http = require('http'), request = require('request'), cheerio = require(
|
||||
'cheerio'), util = require('util');
|
||||
var jsonfile = require('jsonfile'), fs = require('fs'), STRING = require(
|
||||
'string');
|
||||
var converter = require('html-to-markdown');
|
||||
var zlib = require('zlib');
|
||||
var log4js = require('log4js');
|
||||
var logger = log4js.getLogger();
|
||||
var URL = require('url');
|
||||
|
||||
var router = express.Router();
|
||||
|
||||
var EventEmitter = require('events');
|
||||
|
||||
//var nano = require('nano')('http://martind2000:1V3D4m526i@localhost:5984');
|
||||
var busEmitter = new EventEmitter();
|
||||
|
||||
var db_name = 'keeper';
|
||||
//var dbCouch = nano.use(db_name);
|
||||
|
||||
/*
|
||||
|
||||
We've moved to cloudant through IBM Bluemix for the database
|
||||
|
||||
https://25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix.cloudant.com/dashboard.html#usage
|
||||
|
||||
|
||||
*/
|
||||
|
||||
var credentials = {
|
||||
"username": "25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix",
|
||||
"password": "8e417af1b0462ca55726848846cc6b8696fc76defe9d1864cbc334be59549e0c",
|
||||
"host": "25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix.cloudant.com",
|
||||
"port": 443,
|
||||
"url": "https://25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix:8e417af1b0462ca55726848846cc6b8696fc76defe9d1864cbc334be59549e0c@25f854ee-1b51-49ff-acd9-5b0ff478d944-bluemix.cloudant.com",
|
||||
"database" : "keeper"
|
||||
};
|
||||
|
||||
var Cloudant = require('cloudant');
|
||||
var cloudant = Cloudant({account:credentials.username, password:credentials.password});
|
||||
|
||||
var dbCloudant = cloudant.db.use(credentials.database);
|
||||
|
||||
|
||||
var jsonFile = __dirname + '/' + 'output.json';
|
||||
var bodyfile = __dirname + '/' + 'body.html';
|
||||
var htmlfile = __dirname + '/' + 'testoutput.html';
|
||||
var generics = [
|
||||
'ARTICLE',
|
||||
'div.content_column',
|
||||
'div.post',
|
||||
'div.page',
|
||||
'#recipe-single',
|
||||
'div.content.body'
|
||||
];
|
||||
|
||||
var specialHandlers = [{
|
||||
url: 'www.reddit.com', fn: function(body, url) {
|
||||
return doReddit(body, url);
|
||||
}
|
||||
},
|
||||
{
|
||||
url: 'developer.android.com', fn: function(body, url) {
|
||||
return doAndroidDeveloper(body, url);
|
||||
}
|
||||
},
|
||||
{
|
||||
url: 'www.engadget.com', fn: function(body, url) {
|
||||
return doEngadget(body, url);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
];
|
||||
|
||||
|
||||
|
||||
function cleaner(b) {
|
||||
var _b = b;
|
||||
|
||||
var unwanted = [
|
||||
'LINK',
|
||||
'META',
|
||||
'TITLE',
|
||||
'div#disqus_thread',
|
||||
'SCRIPT',
|
||||
'FOOTER',
|
||||
'div.ssba',
|
||||
'.shareaholic-canvas',
|
||||
'.yarpp-related',
|
||||
'div.dfad',
|
||||
'div.postFooterShare',
|
||||
'div#nextPrevLinks',
|
||||
'.post-comments',
|
||||
'HEADER',
|
||||
'.post-title',
|
||||
'#side-menu',
|
||||
'.footer-container',
|
||||
'#pre-footer',
|
||||
'#cakephp-global-navigation',
|
||||
'.masthead',
|
||||
'.breadcrumb-header',
|
||||
'.single-recipe-sidebar',
|
||||
'#recipe-related-videos',
|
||||
'#tnav',
|
||||
'.footer',
|
||||
'#tb-wrapper',
|
||||
'#comments',
|
||||
'#menu'
|
||||
|
||||
];
|
||||
|
||||
for (var i = 0; i < unwanted.length; i++) {
|
||||
_b.find(unwanted[i]).remove();
|
||||
}
|
||||
|
||||
return _b;
|
||||
}
|
||||
|
||||
function insertBookmark(obj) {
|
||||
logger.debug('Inserting into couch...');
|
||||
logger.info(util.inspect(obj));
|
||||
// dbCouch.insert(obj, function(err, body, header) {
|
||||
dbCloudant.insert(obj, function(err, body, header) {
|
||||
if (err) {
|
||||
logger.error('Error inserting into couch');
|
||||
return;
|
||||
}
|
||||
});
|
||||
logger.debug('Insert done..');
|
||||
}
|
||||
|
||||
function updateBookmark(obj, _id, _rev) {
|
||||
logger.debug('Updating couch...');
|
||||
var _obj = obj;
|
||||
_obj._id = _id;
|
||||
_obj._rev = _rev;
|
||||
|
||||
//dbCouch.insert(_obj, function(err, body, header) {
|
||||
dbCloudant.insert(_obj, function(err, body, header) {
|
||||
if (err) {
|
||||
logger.error('Error updating into couch');
|
||||
return;
|
||||
} else {
|
||||
logger.info('I think we updated ok...');
|
||||
busEmitter.emit('updateTagsDB');
|
||||
|
||||
}
|
||||
});
|
||||
logger.debug('Update done..');
|
||||
}
|
||||
var doInsertBookmark = (obj) => {
|
||||
// Logger.info('sendSocket: ' + JSON.stringify(obj));
|
||||
insertBookmark(obj);
|
||||
};
|
||||
|
||||
var doUpdateBookmark = (obj, _id, _rev) => {
|
||||
// Logger.info('sendSocket: ' + JSON.stringify(obj));
|
||||
updateBookmark(obj, _id, _rev);
|
||||
};
|
||||
|
||||
var doGetBookmark = (obj) => {
|
||||
// Logger.info('sendSocket: ' + JSON.stringify(obj));
|
||||
genericGrab(obj);
|
||||
};
|
||||
|
||||
var doGetBookmarkRedo = (obj) => {
|
||||
// Logger.info('sendSocket: ' + JSON.stringify(obj));
|
||||
genericGrab(obj);
|
||||
};
|
||||
|
||||
var doGetBookmarkRes = (url, res) => {
|
||||
logger.debug('doGetBookmarkRes');
|
||||
// Logger.info('sendSocket: ' + JSON.stringify(obj));
|
||||
genericGrab(url, res);
|
||||
};
|
||||
|
||||
var doUpdateTagsDB = () => {
|
||||
logger.debug('Update the tags database...');
|
||||
|
||||
// dbCouch.view('getAllTags', 'getAllTags', function(err, body) {
|
||||
dbCloudant.view('getAllTags', 'getAllTags', function(err, body) {
|
||||
var masterList = [];
|
||||
if (!err) {
|
||||
body.rows.forEach(function(doc) {
|
||||
|
||||
masterList = masterList.concat(doc.value);
|
||||
});
|
||||
|
||||
masterList = masterList.filter((value, index, self) => {
|
||||
return self.indexOf(value) === index;
|
||||
});
|
||||
|
||||
//dbCouch.view('taglist', 'taglist', function(err, body) {
|
||||
dbCloudant.view('taglist', 'taglist', function(err, body) {
|
||||
// Logger.debug(body);
|
||||
if (!err) {
|
||||
|
||||
var outJSON = {};
|
||||
|
||||
body.rows.forEach(function(doc) {
|
||||
doSaveTagsDB(doc.value, masterList);
|
||||
});
|
||||
|
||||
} else {
|
||||
logger.error('NO TAG LIST EXISTS');
|
||||
}
|
||||
});
|
||||
|
||||
} else {
|
||||
|
||||
}
|
||||
});
|
||||
|
||||
};
|
||||
|
||||
var doSaveTagsDB = (orig, newList) => {
|
||||
logger.debug('doSaveTagsDB');
|
||||
|
||||
var _obj = orig;
|
||||
|
||||
_obj.taglist = newList;
|
||||
|
||||
//dbCouch.insert(_obj, function(err, body, header) {
|
||||
dbCloudant.insert(_obj, function(err, body, header) {
|
||||
if (err) {
|
||||
logger.error('Error updating into couch');
|
||||
return;
|
||||
} else {
|
||||
logger.info('Updated the tags list...');
|
||||
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// Events
|
||||
busEmitter.on('saveBookmarkData', doInsertBookmark);
|
||||
busEmitter.on('updateBookmarkData', doUpdateBookmark);
|
||||
busEmitter.on('getBookmark', doGetBookmark);
|
||||
busEmitter.on('getBookmarkRes', doGetBookmarkRes);
|
||||
|
||||
busEmitter.on('getBookmarkRedo', doGetBookmarkRedo);
|
||||
busEmitter.on('updateTagsDB', doUpdateTagsDB);
|
||||
busEmitter.on('saveTagsDB', doSaveTagsDB);
|
||||
|
||||
function doEngadget(body, url) {
|
||||
logger.info('GRABBING Engadget');
|
||||
var obj = {}, tdihbody, i, urlObj, urlPrefix;
|
||||
|
||||
var $ = cheerio.load(body);
|
||||
var title = $('TITLE').text();
|
||||
|
||||
tdihbody = $('DIV#page_body');
|
||||
|
||||
logger.debug('Length:' , tdihbody.length);
|
||||
tdihbody = cleaner(tdihbody);
|
||||
logger.debug('Title: ', title);
|
||||
|
||||
urlObj = URL.parse(url);
|
||||
urlPrefix = urlObj.protocol + '//' + urlObj.host + '/';
|
||||
|
||||
try {
|
||||
tdihbody.find('IMG').each(function(i, elem) {
|
||||
let s, src = $(this).attr('src');
|
||||
|
||||
if (src !== null) {
|
||||
if (!STRING(src).startsWith('http')) {
|
||||
logger.debug('Stripping:' + src);
|
||||
src = urlPrefix + STRING(src).stripLeft('/').trim().s;
|
||||
}
|
||||
|
||||
if (typeof obj.thumbnail === 'undefined') {
|
||||
obj.thumbnail = src;
|
||||
}
|
||||
|
||||
s = 'http://image.silvrtree.co.uk/900,fit/' + src;
|
||||
|
||||
$(this).attr('src', s);
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
catch (e) {
|
||||
logger.error(e);
|
||||
}
|
||||
|
||||
obj.url = STRING(url).trim().s;
|
||||
obj.html = $.html();
|
||||
obj.reduced = STRING(tdihbody.html()).trim().s;
|
||||
obj.nib = STRING(tdihbody.text()).collapseWhitespace().trim().left(300).s;
|
||||
obj.title = STRING(title).collapseWhitespace().s;
|
||||
obj.markdown = converter.convert(obj.reduced);
|
||||
|
||||
return obj;
|
||||
}
|
||||
|
||||
|
||||
function doAndroidDeveloper(body, url) {
|
||||
logger.info('GRABBING AndroidDeveloper');
|
||||
var obj = {}, tdihbody, i, urlObj, urlPrefix;
|
||||
|
||||
var $ = cheerio.load(body);
|
||||
var title = $('TITLE').text();
|
||||
|
||||
tdihbody = $('DIV.jd-descr');
|
||||
|
||||
logger.debug(tdihbody.length);
|
||||
tdihbody = cleaner(tdihbody);
|
||||
logger.debug(title);
|
||||
|
||||
urlObj = URL.parse(url);
|
||||
urlPrefix = urlObj.protocol + '//' + urlObj.host + '/';
|
||||
|
||||
try {
|
||||
tdihbody.find('IMG').each(function(i, elem) {
|
||||
let s, src = $(this).attr('src');
|
||||
|
||||
if (src !== null) {
|
||||
if (!STRING(src).startsWith('http')) {
|
||||
logger.debug('Stripping:' + src);
|
||||
src = urlPrefix + STRING(src).stripLeft('/').trim().s;
|
||||
}
|
||||
|
||||
if (typeof obj.thumbnail === 'undefined') {
|
||||
obj.thumbnail = src;
|
||||
}
|
||||
|
||||
s = 'http://image.silvrtree.co.uk/900,fit/' + src;
|
||||
|
||||
$(this).attr('src', s);
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
catch (e) {
|
||||
logger.error(e);
|
||||
}
|
||||
|
||||
obj.url = STRING(url).trim().s;
|
||||
obj.html = $.html();
|
||||
obj.reduced = STRING(tdihbody.html()).trim().s;
|
||||
obj.nib = STRING(tdihbody.text()).collapseWhitespace().trim().left(300).s;
|
||||
obj.title = STRING(title).collapseWhitespace().s;
|
||||
obj.markdown = converter.convert(obj.reduced);
|
||||
|
||||
return obj;
|
||||
}
|
||||
|
||||
|
||||
function doReddit(body, url) {
|
||||
logger.info('GRABBING REDDIT');
|
||||
var obj = {}, tdihbody, i, urlObj, urlPrefix;
|
||||
|
||||
var $ = cheerio.load(body);
|
||||
var title = $('TITLE').text();
|
||||
|
||||
tdihbody = $('DIV.entry');
|
||||
|
||||
tdihbody.find('A.thumbnail').each(function(i, elem) {
|
||||
|
||||
logger.warn($(this));
|
||||
});
|
||||
|
||||
logger.info('++++++');
|
||||
// Logger.debug(tdihbody.html());
|
||||
|
||||
logger.debug(tdihbody.length);
|
||||
tdihbody = cleaner(tdihbody);
|
||||
logger.debug(title);
|
||||
|
||||
obj.url = STRING(url).trim().s;
|
||||
obj.html = $.html();
|
||||
obj.reduced = STRING(tdihbody.html()).trim().s;
|
||||
obj.nib = STRING(tdihbody.text()).collapseWhitespace().trim().left(300).s;
|
||||
obj.title = STRING(title).collapseWhitespace().s;
|
||||
obj.markdown = converter.convert(obj.reduced);
|
||||
|
||||
return obj;
|
||||
}
|
||||
|
||||
|
||||
|
||||
function genericProcessor(body, url) {
|
||||
logger.info('USING DEFAULT PROCESSOR');
|
||||
var obj = {}, tdihbody, i, urlObj, urlPrefix;
|
||||
var $ = cheerio.load(body);
|
||||
var title = $('TITLE').text();
|
||||
|
||||
i = 0;
|
||||
|
||||
while (($(generics[i]).length == 0) && (i < generics.length)) {
|
||||
i++;
|
||||
}
|
||||
logger.debug(i);
|
||||
|
||||
if (i < generics.length) {
|
||||
logger.warn('Used a generic');
|
||||
tdihbody = $(generics[i]);
|
||||
logger.debug(tdihbody.length);
|
||||
tdihbody = cleaner(tdihbody);
|
||||
logger.debug(title);
|
||||
|
||||
} else {
|
||||
logger.warn('Using whole body');
|
||||
// Bah. nothing to reduce so just grab the body, tidy it and use that
|
||||
tdihbody = $('BODY');
|
||||
|
||||
if (tdihbody.length === 0) {
|
||||
|
||||
tdihbody = $(':root');
|
||||
|
||||
}
|
||||
|
||||
logger.debug(tdihbody.length);
|
||||
tdihbody = cleaner(tdihbody);
|
||||
logger.debug(title);
|
||||
}
|
||||
|
||||
// Logger.info(util.inspect(tdihbody));
|
||||
|
||||
urlObj = URL.parse(url);
|
||||
urlPrefix = urlObj.protocol + '//' + urlObj.host + '/';
|
||||
|
||||
try {
|
||||
tdihbody.find('IMG').each(function(i, elem) {
|
||||
let s, src = $(this).attr('src');
|
||||
|
||||
console.log('!!!!' + src);
|
||||
if (src !== null && typeof src !== 'undefined') {
|
||||
if (!STRING(src).startsWith('http')) {
|
||||
logger.debug('Stripping:' + src);
|
||||
src = urlPrefix + STRING(src).stripLeft('/').trim().s;
|
||||
}
|
||||
|
||||
if (typeof obj.thumbnail === 'undefined') {
|
||||
obj.thumbnail = src;
|
||||
}
|
||||
|
||||
s = 'http://image.silvrtree.co.uk/900,fit/' + src;
|
||||
|
||||
$(this).attr('src', s);
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
catch (e) {
|
||||
logger.error(e);
|
||||
}
|
||||
|
||||
obj.url = STRING(url).trim().s;
|
||||
obj.html = $.html();
|
||||
obj.reduced = STRING(tdihbody.html()).trim().s;
|
||||
obj.nib = STRING(tdihbody.text()).collapseWhitespace().trim().left(300).s;
|
||||
obj.title = STRING(title).collapseWhitespace().s;
|
||||
obj.markdown = converter.convert(obj.reduced);
|
||||
|
||||
|
||||
return obj;
|
||||
}
|
||||
function processBody(body, url, _id, _rev) {
|
||||
|
||||
var obj = {}, i, urlObj, urlPrefix;
|
||||
|
||||
|
||||
|
||||
|
||||
// Try to find a body to grab
|
||||
|
||||
urlObj = URL.parse(url);
|
||||
|
||||
logger.debug('host:', urlObj.host);
|
||||
|
||||
var flag;
|
||||
for (i = 0;i < specialHandlers.length;i++) {
|
||||
if (urlObj.host === specialHandlers[i].url) {
|
||||
flag = true;
|
||||
obj = specialHandlers[i].fn(body,url);
|
||||
}
|
||||
}
|
||||
|
||||
if (!flag) {
|
||||
// Do generic processing
|
||||
obj = genericProcessor(body,url);
|
||||
}
|
||||
|
||||
// Logger.warn(obj.reduced);
|
||||
|
||||
obj.host = urlObj.host;
|
||||
|
||||
|
||||
/* Jsonfile.writeFile(jsonFile, obj, function (err) {
|
||||
console.error(err);
|
||||
});*/
|
||||
|
||||
if (_id !== null) {
|
||||
busEmitter.emit('updateBookmarkData', obj, _id, _rev);
|
||||
} else {
|
||||
busEmitter.emit('saveBookmarkData', obj);
|
||||
}
|
||||
|
||||
return obj;
|
||||
|
||||
}
|
||||
function genericGrab(obj, res) {
|
||||
|
||||
var url, _id = null, _ver = null;
|
||||
|
||||
if (typeof obj === 'string') {
|
||||
logger.info(obj);
|
||||
url = obj;
|
||||
} else {
|
||||
url = obj.url;
|
||||
_id = obj._id || null;
|
||||
_ver = obj._rev || null;
|
||||
}
|
||||
|
||||
logger.warn(typeof obj);
|
||||
|
||||
logger.info(url);
|
||||
logger.info(_id);
|
||||
logger.info(_ver);
|
||||
|
||||
var options = {
|
||||
url: url,
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36'
|
||||
},
|
||||
jar: true,
|
||||
followRedirect: true,
|
||||
followAllRedirects: true
|
||||
};
|
||||
|
||||
request(options, function(err, resp, body) {
|
||||
if (err)
|
||||
throw err;
|
||||
|
||||
if (resp.headers.hasOwnProperty('content-encoding')) {
|
||||
logger.warn('content-encoding');
|
||||
if (resp.headers['content-encoding'] == 'gzip') {
|
||||
|
||||
// to test http://chaosinthekitchen.com/2009/07/lime-and-coconut-chicken/
|
||||
|
||||
var gunzip = zlib.createGunzip();
|
||||
var jsonString = '';
|
||||
resp.pipe(gunzip);
|
||||
gunzip.on('data', function(chunk) {
|
||||
jsonString += chunk;
|
||||
});
|
||||
gunzip.on('end', function() {
|
||||
// Console.log((jsonString));
|
||||
callback(JSON.stringify(jsonString));
|
||||
});
|
||||
gunzip.on('error', function(e) {
|
||||
console.log(e);
|
||||
});
|
||||
} else {
|
||||
var b = processBody(body, url, _id, _ver);
|
||||
if (res != null) {
|
||||
res.render('grabbed');
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
var b = processBody(body, url, _id, _ver);
|
||||
if (res != null) {
|
||||
res.render('grabbed', {data: b});
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
router.get('/pocket', function(req, res) {
|
||||
logger.debug('list..');
|
||||
|
||||
|
||||
// dbCouch.view('pocketList', 'pocketList', function(err, body) {
|
||||
dbCloudant.view('pocketList', 'pocketList', function(err, body) {
|
||||
if (!err) {
|
||||
|
||||
var outJSON = [];
|
||||
body.rows.forEach(function(doc) {
|
||||
var obj = {id: doc.id, entry: doc.value};
|
||||
console.log(typeof obj.entry.tn);
|
||||
if (typeof obj.entry.tn === 'string') {
|
||||
console.log('its a string:', typeof obj.entry.tn)
|
||||
obj.entry.tn = 'http://image.silvrtree.co.uk/100,fit,q80/' + obj.entry.tn;
|
||||
} else {
|
||||
obj.entry.tn = 'gfx/fm.png';
|
||||
}
|
||||
|
||||
outJSON.push(obj);
|
||||
|
||||
|
||||
});
|
||||
|
||||
logger.debug(util.inspect(body));
|
||||
logger.info(util.inspect(outJSON));
|
||||
res.render('pocket', {data: outJSON});
|
||||
} else {
|
||||
res.writeHead(500, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({}));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
|
||||
router.get('/list', function(req, res) {
|
||||
logger.debug('list..');
|
||||
|
||||
//dbCouch.view('titles', 'titles', function(err, body) {
|
||||
dbCloudant.view('titles', 'titles', function(err, body) {
|
||||
if (!err) {
|
||||
|
||||
var outJSON = [];
|
||||
body.rows.forEach(function(doc) {
|
||||
outJSON.push({id: doc.id, title: doc.value});
|
||||
});
|
||||
|
||||
//Logger.debug(util.inspect(body));
|
||||
res.writeHead(200, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({list: outJSON}));
|
||||
|
||||
} else {
|
||||
res.writeHead(500, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({}));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
router.get('/entry/:id', function(req, res) {
|
||||
logger.debug('entry..');
|
||||
|
||||
logger.debug(req.params.id);
|
||||
|
||||
//dbCouch.get(req.params.id, function(err, body) {
|
||||
dbCloudant.get(req.params.id, function(err, body) {
|
||||
if (!err) {
|
||||
|
||||
var outJSON = {};
|
||||
outJSON._id = body._id;
|
||||
outJSON._rev = body._rev;
|
||||
outJSON.title = body.title;
|
||||
outJSON.reduced = body.reduced;
|
||||
outJSON.url = body.url;
|
||||
outJSON.tags = body.tags || {solid: '', list: []};
|
||||
|
||||
//Logger.debug(util.inspect(body));
|
||||
res.writeHead(200, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify(outJSON));
|
||||
|
||||
} else {
|
||||
res.writeHead(500, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({}));
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
router.route('/tags')
|
||||
.get(function(req, res, next) {
|
||||
logger.debug('tag list..');
|
||||
|
||||
logger.debug(req.params.id);
|
||||
|
||||
//dbCouch.view('taglist', 'taglist', function(err, body) {
|
||||
dbCloudant.view('taglist', 'taglist', function(err, body) {
|
||||
if (!err) {
|
||||
logger.debug(body);
|
||||
var outJSON = [];
|
||||
body.rows.forEach(function(doc) {
|
||||
logger.info(doc.value.taglist);
|
||||
if (doc.value[0] == req.params.id) {
|
||||
outJSON = doc.value.taglist.sort();
|
||||
}
|
||||
});
|
||||
|
||||
//Logger.debug(util.inspect(body));
|
||||
res.writeHead(200, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({list: outJSON}));
|
||||
|
||||
} else {
|
||||
logger.error(err);
|
||||
res.writeHead(500, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({}));
|
||||
}
|
||||
});
|
||||
|
||||
}).post(function(req, res, next) {
|
||||
var t = req.body;
|
||||
console.log(t);
|
||||
|
||||
logger.info('regetting:' + req.body._id);
|
||||
|
||||
//dbCouch.get(req.body._id, function(err, body) {
|
||||
dbCloudant.get(req.body._id, function(err, body) {
|
||||
if (!err) {
|
||||
|
||||
var obj = {};
|
||||
|
||||
obj.url = body.url;
|
||||
obj.html = body.html;
|
||||
obj.reduced = body.reduced;
|
||||
obj.title = body.title;
|
||||
obj.tags = req.body.tags;
|
||||
|
||||
logger.info('Updating...');
|
||||
busEmitter.emit('updateBookmarkData', obj, body._id, body._rev, res);
|
||||
|
||||
var outJSON = {};
|
||||
outJSON._id = body._id;
|
||||
outJSON._rev = body._rev;
|
||||
outJSON.title = body.title;
|
||||
outJSON.reduced = body.reduced;
|
||||
outJSON.url = body.url;
|
||||
outJSON.tags = req.body.tags;
|
||||
|
||||
//Logger.debug(util.inspect(body));
|
||||
res.writeHead(200, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify(outJSON));
|
||||
|
||||
} else {
|
||||
res.writeHead(500, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({}));
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
|
||||
router.get('/tags/:id', function(req, res) {
|
||||
logger.debug('entry..');
|
||||
|
||||
logger.debug(req.params.id);
|
||||
|
||||
//dbCouch.view('getTagByKey', 'getTagByKey', function(err, body) {
|
||||
dbCloudant.view('getTagByKey', 'getTagByKey', function(err, body) {
|
||||
if (!err) {
|
||||
// Logger.debug(body);
|
||||
var outJSON = [];
|
||||
body.rows.forEach(function(doc) {
|
||||
// Logger.debug(doc);
|
||||
if (doc.value[0] == req.params.id) {
|
||||
outJSON.push({id: doc.id, title: doc.value[1]})
|
||||
}
|
||||
});
|
||||
|
||||
//Logger.debug(util.inspect(body));
|
||||
res.writeHead(200, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({list: outJSON}));
|
||||
|
||||
} else {
|
||||
logger.error(err);
|
||||
res.writeHead(500, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({}));
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
router.post('/add', function(req, res) {
|
||||
logger.debug('add entry..');
|
||||
|
||||
var t = req.body;
|
||||
if (t.hasOwnProperty('url')) {
|
||||
var url = JSON.parse(t.url.toString());
|
||||
logger.debug(url);
|
||||
busEmitter.emit('getBookmark', t);
|
||||
} else {
|
||||
logger.error('No data block!');
|
||||
}
|
||||
res.writeHead(200, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({adding: url}));
|
||||
|
||||
});
|
||||
|
||||
|
||||
router.post('/redo', function(req, res) {
|
||||
logger.debug('redoing entry..');
|
||||
|
||||
var t = req.body;
|
||||
console.log(t);
|
||||
if (t.hasOwnProperty('url')) {
|
||||
var url = t.url.toString();
|
||||
logger.debug(url);
|
||||
busEmitter.emit('getBookmark', t);
|
||||
} else {
|
||||
logger.error('No data block!');
|
||||
}
|
||||
res.writeHead(200, {ContentType: 'application/json'});
|
||||
res.end(JSON.stringify({adding: url}));
|
||||
|
||||
});
|
||||
|
||||
router.route('/new')
|
||||
.get(function(req, res, next) {
|
||||
logger.debug('Save new');
|
||||
busEmitter.emit('getBookmarkRes', req.query.url, res);
|
||||
}).post(function(req, res, next) {
|
||||
logger.debug('Posted Save new');
|
||||
logger.info(req.body);
|
||||
if (Object.keys(req.body).length !== 0) {
|
||||
busEmitter.emit('getBookmarkRes', req.body.url, res);
|
||||
} else {
|
||||
res.status(422).end();
|
||||
}
|
||||
});
|
||||
|
||||
busEmitter.emit('updateTagsDB');
|
||||
|
||||
module.exports = router;
|
Loading…
Reference in New Issue
Block a user