keeper/server/keeper.js

"use strict";
/**
 * Created by Martin on 22/02/2016.
 */
var express = require('express');
var http = require('http'), request = require('request'), cheerio = require('cheerio'), util = require('util');
var jsonfile = require('jsonfile'), fs = require('fs'), STRING = require('string');
var zlib = require("zlib");
var log4js = require('log4js');
var logger = log4js.getLogger();

var router = express.Router();

var EventEmitter = require('events');

var nano = require('nano')('http://localhost:5984');
var busEmitter = new EventEmitter();

var db_name = 'keeper';
var dbCouch = nano.use(db_name);


var bodyfile = __dirname + '/' + 'body.html';
var htmlfile = __dirname + '/' + 'testoutput.html';
var generics = ['ARTICLE', 'div.content_column', 'div.post'];


function cleaner(b) {
    var _b = b;

    var unwanted = ['div#disqus_thread', 'SCRIPT', 'FOOTER', 'div.ssba', '.shareaholic-canvas', '.yarpp-related', 'div.dfad', 'div.postFooterShare', 'div#nextPrevLinks', '.post-comments'];

    for (var i = 0; i < unwanted.length; i++) {
        _b.find(unwanted[i]).remove();
    }
    return _b;
}

function insertBookmark(obj) {
  logger.debug('Inserting into couch...');
  logger.info(util.inspect(obj));
  dbCouch.insert(obj, function(err, body,header) {
    if (err) {
      logger.error('Error inserting into couch');
      return;
    }
  });
    logger.debug('Insert done..');
}

var doInsertBookmark = (obj) =>{
    //   logger.info('sendSocket: ' + JSON.stringify(obj));
    insertBookmark(obj);
};

var doGetBookmark = (url) =>{
    //   logger.info('sendSocket: ' + JSON.stringify(obj));
    genericGrab(url);
};

var doGetBookmarkRes = (url,res) =>{
  logger.debug('doGetBookmarkRes');
  //   logger.info('sendSocket: ' + JSON.stringify(obj));
  genericGrab(url,res);
};

// Events
busEmitter.on('saveBookmarkData', doInsertBookmark);
busEmitter.on('getBookmark', doGetBookmark);
busEmitter.on('getBookmarkRes', doGetBookmarkRes);

function processBody(body,url) {

   var $ = cheerio.load(body);
    var title = $('TITLE').text();

    // try to find a body to grab

    var i = 0;

    while (($(generics[i]).length == 0) && (i < generics.length)) {
     //   logger.info(generics[i]);
     //   logger.info($(generics[i]));
     //   logger.info('i: ' + i + ', ' + $(generics[i]).length);
        i++;
    }
    logger.debug(i);

    if (i < generics.length) {
        var tdihbody = $(generics[i]);
        var obj = {};

        logger.debug(tdihbody.length);
        tdihbody = cleaner(tdihbody);
        logger.debug(title);

        obj.url=url;
        obj.html = $.html();
        obj.reduced = STRING(tdihbody.html()).collapseWhitespace().s;
        obj.title = STRING(title).collapseWhitespace().s;

        busEmitter.emit("saveBookmarkData", obj);

      return obj;
    }


}
function genericGrab(url,res) {
    logger.info(url);
    request(url, function (err, resp, body) {
        if (err)
            throw err;

        if (resp.headers.hasOwnProperty('content-encoding')) {
          logger.warn('content-encoding');
            if (resp.headers['content-encoding'] == 'gzip') {

                // to test http://chaosinthekitchen.com/2009/07/lime-and-coconut-chicken/


                var gunzip = zlib.createGunzip();
                var jsonString = '';
                resp.pipe(gunzip);
                gunzip.on('data', function (chunk) {
                    jsonString += chunk;
                });
                gunzip.on('end', function () {
                    console.log((jsonString));
                    callback(JSON.stringify(jsonString));
                });
                gunzip.on('error', function (e) {
                    console.log(e);
                });
            }
            else
            {
               var b = processBody(body,url);
              if (res != null)
              {
                res.render('grabbed');
              }
            }

        } else
        {
          var b = processBody(body,url);
          if (res != null)
          {
            res.render('grabbed',{data:b});
          }
        }

    });
}

router.get('/list', function (req, res) {
    logger.debug('list..');

  dbCouch.view('titles','titles',function(err, body) {
    if (!err) {

      var outJSON = [];
      body.rows.forEach(function(doc) {
        outJSON.push({id:doc.id, title:doc.value })
      });

      //logger.debug(util.inspect(body));
      res.writeHead(200, {"ContentType": "application/json"});
      res.end(JSON.stringify({list: outJSON}));

    } else
    {
      res.writeHead(500, {"ContentType": "application/json"});
      res.end(JSON.stringify({}));
    }
  });
});

router.get('/entry/:id', function (req, res) {
  logger.debug('entry..');

  logger.debug(req.params.id);

  dbCouch.get(req.params.id,function(err, body) {
    if (!err) {

      var outJSON = {};
      outJSON.title = body.title;
      outJSON.reduced = body.reduced;
      //logger.debug(util.inspect(body));
      res.writeHead(200, {"ContentType": "application/json"});
      res.end(JSON.stringify(outJSON));

    } else
    {
      res.writeHead(500, {"ContentType": "application/json"});
      res.end(JSON.stringify({}));
    }
  });

});

router.post('/add', function (req, res) {
    logger.debug('add entry..');

    var t = req.body;
    if (t.hasOwnProperty('url')) {
        var url = JSON.parse(t.url.toString());
        logger.debug(url);
        busEmitter.emit("getBookmark", url);
    }
    else {
        logger.error('No data block!');
    }
    res.writeHead(200, {"ContentType": "application/json"});
    res.end(JSON.stringify({adding: url}));

});

router.get('/new', function (req, res) {
  logger.debug('Save new');
  busEmitter.emit("getBookmarkRes", req.query.url ,res);

});

module.exports = router;