initial

.gitignore

@@ -0,0 +1,78 @@
+# Created by .ignore support plugin (hsz.mobi)
+### JetBrains template
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio
+
+*.iml
+
+## Directory-based project format:
+.idea/
+# if you remove the above rule, at least ignore the following:
+
+# User-specific stuff:
+# .idea/workspace.xml
+# .idea/tasks.xml
+# .idea/dictionaries
+
+# Sensitive or high-churn files:
+# .idea/dataSources.ids
+# .idea/dataSources.xml
+# .idea/sqlDataSources.xml
+# .idea/dynamic.xml
+# .idea/uiDesigner.xml
+
+# Gradle:
+# .idea/gradle.xml
+# .idea/libraries
+
+# Mongo Explorer plugin:
+# .idea/mongoSettings.xml
+
+## File-based project format:
+*.ipr
+*.iws
+
+## Plugin-specific files:
+
+# IntelliJ
+/out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+### Node template
+# Logs
+logs
+*.log
+npm-debug.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directory
+# https://docs.npmjs.com/misc/faq#should-i-check-my-node-modules-folder-into-git
+node_modules
+

package.json

@@ -0,0 +1,25 @@
+{
+  "name": "recipe",
+  "version": "1.0.0",
+  "description": "",
+  "main": "recipe-server.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "body-parser": "^1.15.0",
+    "cheerio": "^0.20.0",
+    "cookie-parser": "^1.4.1",
+    "express": "^4.13.4",
+    "morgan": "^1.7.0",
+    "serve-favicon": "^2.3.0",
+    "string": "^3.3.1"
+  },
+  "devDependencies": {
+    "jsonfile": "^2.2.3",
+    "log4js": "^0.6.31",
+    "simplecrawler": "^0.6.2"
+  }
+}

recipe-server.js

@@ -0,0 +1,46 @@
+var express = require('express'), path = require('path'), http = require('http'),
+
+
+    favicon = require('serve-favicon'),
+    logger = require('morgan'),
+    cookieParser = require('cookie-parser'),
+    bodyParser = require('body-parser'),
+
+    routes = require('./routes/index'),
+    users = require('./routes/users')
+
+
+//train = require('lib/train')
+/* ,submit = require('./routes/mongo/submit') */
+    ;
+var app = express();
+
+app.set('port', process.env.PORT || 8025);
+app.set('views', path.join(__dirname, 'views'));
+app.set('view engine', 'ejs');
+app.use(logger('dev'));
+app.use(bodyParser.json());
+app.use(bodyParser.urlencoded({ extended: true }));
+app.use(cookieParser());
+app.use(express.static(path.join(__dirname, 'public')));
+
+app.use('/', routes);
+//app.use('/users', users);
+/*
+
+app.use('/time', timeroute);
+app.use('/btc', btcroute);
+app.use('/temp',temproute);
+app.use('/weight',weightroute);
+*/
+
+
+/**
+ * create the server
+ */
+
+app.listen(app.get('port'), function () {
+    console.log('Recipe Server listening on ' + app.get('port'));
+});/**
+ * Created by Martin on 22/02/2016.
+ */

recipes.json

@@ -0,0 +1,49 @@
+{"recipes":[
+    {"url":"", "title":""},
+
+
+    {"url":"http://www.simplyrecipes.com/recipes/grilled_lime_chicken_with_black_bean_sauce/", "title":"Grilled Lime Chicken with Black Bean Sauce Recipe"},
+    {"url":"http://www.marksdailyapple.com/shakshuka-eggs-poached-in-spicy-tomato-sauce/#axzz29jTSubMo", "title":"Shakshuka (Eggs Poached in Spicy Tomato Sauce) "},
+    {"url":"http://www.marksdailyapple.com/spiced-pork-and-butternut-squash-with-sage/#axzz29jTSubMo", "title":"Spiced Pork and Butternut Squash with Sage"},
+    {"url":"http://www.marksdailyapple.com/dairy-free-green-goddess-dressing/#axzz29jTSubMo", "title":"Dairy-Free Green Goddess Dressing"},
+    {"url":"http://www.marksdailyapple.com/pork-stuffed-jalapeno-peppers/#axzz29jTSubMo", "title":"Pork-Stuffed Jalapeño Peppers"},
+    {"url":"http://www.marksdailyapple.com/herb-chicken-cooked-under-a-brick/#axzz29jTSubMo", "title":"Herb Chicken Cooked Under a Brick"},
+    {"url":"http://www.marksdailyapple.com/balsamic-glazed-drumsticks/#axzz29jTSubMo", "title":"Balsamic-Glazed Drumsticks"},
+    {"url":"http://www.marksdailyapple.com/slow-cooked-coconut-ginger-pork/#axzz29jTSubMo", "title":"Slow-Cooked Coconut Ginger Pork"},
+    {"url":"http://www.marksdailyapple.com/lime-and-basil-beef-kebabs/#axzz29jTSubMo", "title":"Lime and Basil Beef Kebabs"},
+    {"url":"http://www.marksdailyapple.com/taco-bowl-with-crispy-kale-chips/#axzz29jTSubMo", "title":"Taco Bowl with Crispy Kale Chips"},
+    {"url":"http://www.marksdailyapple.com/grilled-eggs-with-mexican-chorizo/#axzz29jTSubMo", "title":"Grilled Eggs with Mexican Chorizo"},
+    {"url":"http://www.marksdailyapple.com/banh-mi-salad/#axzz29jTSubMo", "title":"Banh Mi Salad"},
+    {"url":"http://www.marksdailyapple.com/tender-lemon-parsley-brisket/#axzz29jTSubMo", "title":"Tender Lemon-Parsley Brisket"},
+    {"url":"http://www.marksdailyapple.com/butter-stuffed-chicken-kiev/#axzz29jTSubMo", "title":"Butter-Stuffed Chicken Kiev"},
+    {"url":"http://www.marksdailyapple.com/primal-chicken-tikka-masala/#axzz29jTSubMo", "title":"Primal Chicken Tikka Masala"},
+    {"url":"http://www.nerdfitness.com/blog/2012/02/21/how-to-cook-paleo-spaghetti/", "title":"Paleo Spaghetti"},
+    {"url":"http://www.nerdfitness.com/blog/2011/02/21/a-decent-meal/", "title":"How to Grow Up And Cook a Decent Meal"},
+    {"url":"http://www.marksdailyapple.com/fajita-frittata-with-avocado-salsa/#axzz29jTSubMo", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+    {"url":"", "title":""},
+]}

server/grab.js

@@ -0,0 +1,48 @@
+/**
+ * Created by Martin on 22/02/2016.
+ */
+
+var http = require('http'), request = require('request'), cheerio = require('cheerio'), util = require('util');
+var jsonfile = require('jsonfile'), fs = require('fs'), STRING = require('string');
+var log4js = require('log4js');
+var logger = log4js.getLogger();
+
+var bodyfile = __dirname + '/' + 'body.html';
+var htmlfile = __dirname + '/' + 'testoutput.html';
+
+module.exports = {
+
+    grab:function(url,section) {
+        logger.info(url);
+        request(url, function (err, resp, body) {
+            if (err)
+                throw err;
+
+            $ = cheerio.load(body);
+            var tdihbody = $(section);
+            var
+
+            fs.writeFileSync(bodyfile, $.html());
+            fs.writeFileSync(htmlfile, tdihbody.html());
+
+        });
+    },
+    grabMarksDailyApple:function(url,section) {
+        logger.info(url);
+        var section = '#contentColumn';
+        request(url, function (err, resp, body) {
+            if (err)
+                throw err;
+
+            $ = cheerio.load(body);
+            var tdihbody = $(section);
+            var
+
+                fs.writeFileSync(bodyfile, $.html());
+            fs.writeFileSync(htmlfile, tdihbody.html());
+
+        });
+    }
+};
+
+module.exports.grabMarksDailyApple('http://www.marksdailyapple.com/shakshuka-eggs-poached-in-spicy-tomato-sauce');

server/simplecrawler.js

@@ -0,0 +1,52 @@
+var Crawler = require("simplecrawler"),fs = require('fs');
+
+
+//var myCrawler = new Crawler("http://www.bbc.co.uk/food/recipes/chicken_piperade_with_23608");
+
+var myCrawler = new Crawler("www.bbc.co.uk", "/food/recipes/chicken_piperade_with_23608", 80);
+
+var htmlfile = __dirname + '/' + 'test.html';
+
+myCrawler.maxDepth = 1;
+//myCrawler.interval = 10000; // Ten seconds
+myCrawler.maxConcurrency = 1;
+
+
+myCrawler.on('crawlstart', function() {
+    console.log('Crawling started...');
+});
+
+myCrawler.on('fetchstart  ', function(a, b) {
+    console.log('fetchstart ...');
+    console.log(a);
+    console.log(b);
+});
+
+myCrawler.on('fetcherror ', function(a, b) {
+    console.log('Crawling error...');
+    console.log(a);
+    console.log(b);
+});
+
+myCrawler.on('fetchclienterror  ', function(a, b) {
+    console.log('fetchclienterror  error...');
+    console.log(a);
+    console.log(b);
+});
+
+myCrawler.on('queueadd  ', function(a) {
+    console.log('fetchclienterror  error...');
+    console.log(a);
+
+});
+
+myCrawler.on("fetchcomplete", function(queueItem, responseBuffer, response) {
+    console.log("I just received %s (%d bytes)", queueItem.url, responseBuffer.length);
+    console.log("It was a resource of type %s", response.headers['content-type']);
+
+    // Do something with the data in responseBuffer
+
+    fs.writeFileSync(htmlfile, responseBuffer);
+});
+
+myCrawler.start();