mirror of
https://gitlab.silvrtree.co.uk/martind2000/recipes.git
synced 2025-03-13 09:20:00 +00:00
trying to add gzip decompression
This commit is contained in:
Martin Donnelly
parent
fb7ccf8230
commit
0e3f85240e
66
app/index.prod.html
Normal file
66
app/index.prod.html
Normal file
@ -0,0 +1,66 @@
|
||||
<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<link href="//fonts.googleapis.com/css?family=Roboto+Slab:400,300,700" rel="stylesheet" type="text/css">
|
||||
<link rel="stylesheet" href="//cdn.linearicons.com/free/1.0.0/icon-font.min.css">
|
||||
<link href="//cdn.muicss.com/mui-0.4.6/css/mui.min.css" rel="stylesheet" type="text/css" />
|
||||
<!-- inject:css -->
|
||||
<link href="css/app.min.css" rel="stylesheet" type="text/css" />
|
||||
<link href="css/md.min.css" rel="stylesheet" type="text/css" />
|
||||
<!-- endinject -->
|
||||
<script src="//cdn.muicss.com/mui-0.4.6/js/mui.min.js"></script>
|
||||
<script src="//code.jquery.com/jquery-2.1.4.min.js"></script>
|
||||
<script src="libs/ejs_production.js"></script>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
<div id="sidedrawer" class="mui--no-user-select">
|
||||
<div id="sidedrawer-brand" class="mui--appbar-line-height mui--text-title">Recipes</div>
|
||||
<div class="mui-divider"></div>
|
||||
<div id="functions" class="fnBlock">
|
||||
<span id="fnRefresh" class="lnr lnr-sync"></span>
|
||||
<span id="fnSearch" class="lnr lnr-magnifier"></span>
|
||||
</div>
|
||||
<div id='searchbox' class="mui-textfield" style="display: none;">
|
||||
<input id='newsearch' type="text" placeholder="Search for..">
|
||||
</div>
|
||||
<div class="mui-divider"></div>
|
||||
<div id="listContainer">
|
||||
|
||||
</div>
|
||||
<div class="mui-divider"></div>
|
||||
<div class="mui-textfield">
|
||||
<input id='newurl' type="text" placeholder="Add new url">
|
||||
<div id="addstatus" style="display:none;">Adding...</div>
|
||||
</div>
|
||||
</div>
|
||||
<header id="header">
|
||||
<div class="mui-appbar mui--appbar-line-height">
|
||||
<div class="mui-container-fluid">
|
||||
<a class="sidedrawer-toggle mui--visible-xs-inline-block js-show-sidedrawer">☰</a>
|
||||
<a class="sidedrawer-toggle mui--hidden-xs js-hide-sidedrawer">☰</a>
|
||||
<span class="mui--text-title mui--visible-xs-inline-block">Brand.io</span>
|
||||
</div>
|
||||
</div>
|
||||
</header>
|
||||
<div id="content-wrapper">
|
||||
<div class="mui--appbar-height"></div>
|
||||
<div class="mui-container-fluid" id="bodyContents">
|
||||
<!-- content here -->
|
||||
</div>
|
||||
</div>
|
||||
<footer id="footer">
|
||||
<div class="mui-container-fluid">
|
||||
<br>
|
||||
Made with ♥ by Martin</a>
|
||||
</div>
|
||||
</footer>
|
||||
</body>
|
||||
<!-- inject:js -->
|
||||
<script src="js/main.min.js"></script>
|
||||
|
||||
<!-- endinject -->
|
||||
</html>
|
||||
18
gulpfile.js
18
gulpfile.js
@ -46,22 +46,10 @@ gulp.task('partials', function() {
|
||||
});
|
||||
|
||||
gulp.task('minify-html', function () {
|
||||
var sources = gulp.src(['dist/**/*.min.js', 'dist/**/*.min.css'], {read: false});
|
||||
var appstream = gulp.src(['dist/**/*.min.js'], {read:false});
|
||||
return gulp.src(['app/*.html']).pipe(inject(gulp.src(['./dist/js/**/*.min.js', './dist/css/**/*.min.css'], {read: false}), {
|
||||
starttag: '"main": [',
|
||||
endtag: ']',
|
||||
transform: function (filepath, file, i, length) {
|
||||
return ' "' + filepath + '"' + (i + 1 < length ? ',' : '');
|
||||
}
|
||||
}))
|
||||
.pipe(gulp.dest('dist'));
|
||||
/*.pipe(gulp.dest('./'));
|
||||
.pipe(inject(sources),{relative:true})
|
||||
/!*.pipe(htmlmin({removeComments: true, collapseWhitespace: true, keepClosingSlash: true}))*!/
|
||||
.pipe(gulp.dest('dist'));*/
|
||||
});
|
||||
|
||||
return gulp.src(['app/index.prod.html']).pipe(htmlmin({removeComments: true, collapseWhitespace: true, keepClosingSlash: true}))
|
||||
.pipe(gulp.dest('dist/index.html'));
|
||||
});
|
||||
|
||||
gulp.task('clean', function() {
|
||||
return del(['dist']);
|
||||
|
||||
@ -17,7 +17,8 @@
|
||||
"require-dir": "^0.3.0",
|
||||
"serve-favicon": "^2.3.0",
|
||||
"sqlite3": "^3.1.1",
|
||||
"string": "^3.3.1"
|
||||
"string": "^3.3.1",
|
||||
"zlib": "^1.0.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"del": "^2.2.0",
|
||||
|
||||
BIN
server/body.html
BIN
server/body.html
Binary file not shown.
@ -1,9 +1,11 @@
|
||||
"use strict";
|
||||
/**
|
||||
* Created by Martin on 22/02/2016.
|
||||
*/
|
||||
var express = require('express');
|
||||
var http = require('http'), request = require('request'), cheerio = require('cheerio'), util = require('util');
|
||||
var jsonfile = require('jsonfile'), fs = require('fs'), STRING = require('string');
|
||||
var zlib = require("zlib");
|
||||
var log4js = require('log4js');
|
||||
var logger = log4js.getLogger();
|
||||
|
||||
@ -93,40 +95,79 @@ busEmitter.on('saveRecipeData', doInsertRecipe);
|
||||
|
||||
busEmitter.on('getRecipe', doGetRecipe);
|
||||
|
||||
function processBody(body) {
|
||||
|
||||
$ = cheerio.load(body);
|
||||
var title = $('TITLE').text();
|
||||
|
||||
// try to find a body to grab
|
||||
|
||||
var i = 0;
|
||||
|
||||
while (($(generics[i]).length == 0) && (i < generics.length)) {
|
||||
logger.info(generics[i]);
|
||||
logger.info($(generics[i]));
|
||||
logger.info('i: ' + i + ', ' + $(generics[i]).length);
|
||||
i++;
|
||||
}
|
||||
logger.debug(i);
|
||||
|
||||
if (i < generics.length) {
|
||||
var tdihbody = $(generics[i]);
|
||||
var obj = [];
|
||||
|
||||
logger.debug(tdihbody.length);
|
||||
tdihbody = cleaner(tdihbody);
|
||||
logger.debug(title);
|
||||
|
||||
obj.push(url);
|
||||
obj.push($.html());
|
||||
obj.push(tdihbody.html());
|
||||
obj.push(title);
|
||||
// busEmitter.emit("saveRecipeData", obj);
|
||||
|
||||
}
|
||||
}
|
||||
function genericGrab(url) {
|
||||
logger.info(url);
|
||||
request(url, function (err, resp, body) {
|
||||
if (err)
|
||||
throw err;
|
||||
|
||||
$ = cheerio.load(body);
|
||||
var title = $('TITLE').text();
|
||||
console.log("headers: ", resp.headers);
|
||||
console.log(resp.statusCode);
|
||||
|
||||
// try to find a body to grab
|
||||
if (resp.headers.hasOwnProperty('content-encoding')) {
|
||||
if (resp.headers['content-encoding'] == 'gzip') {
|
||||
|
||||
var i = 0;
|
||||
// to test http://chaosinthekitchen.com/2009/07/lime-and-coconut-chicken/
|
||||
|
||||
while (($(generics[i]).length == 0) && (i < generics.length)) {
|
||||
|
||||
i++;
|
||||
var gunzip = zlib.createGunzip();
|
||||
var jsonString = '';
|
||||
resp.pipe(gunzip);
|
||||
gunzip.on('data', function (chunk) {
|
||||
jsonString += chunk;
|
||||
});
|
||||
gunzip.on('end', function () {
|
||||
console.log((jsonString));
|
||||
callback(JSON.stringify(jsonString));
|
||||
});
|
||||
gunzip.on('error', function (e) {
|
||||
console.log(e);
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
processBody(body);
|
||||
}
|
||||
|
||||
} else
|
||||
{
|
||||
processBody(body);
|
||||
}
|
||||
logger.debug(i);
|
||||
|
||||
if (i < generics.length) {
|
||||
var tdihbody = $(generics[i]);
|
||||
var obj = [];
|
||||
|
||||
logger.debug(tdihbody.length);
|
||||
tdihbody = cleaner(tdihbody);
|
||||
logger.debug(title);
|
||||
|
||||
obj.push(url);
|
||||
obj.push($.html());
|
||||
obj.push(tdihbody.html());
|
||||
obj.push(title);
|
||||
busEmitter.emit("saveRecipeData", obj);
|
||||
|
||||
}
|
||||
//fs.writeFileSync(htmlfile, tdihbody.html());
|
||||
fs.writeFileSync(bodyfile, $.html());
|
||||
});
|
||||
@ -213,6 +254,6 @@ router.post('/add', function (req, res) {
|
||||
|
||||
module.exports = router;
|
||||
|
||||
createDB();
|
||||
//createDB();
|
||||
//module.exports.grabMarksDailyApple('http://www.marksdailyapple.com/spiced-pork-and-butternut-squash-with-sage');
|
||||
//module.exports.generic('http://www.health-bent.com/soups/paleo-mediterranean-beef-stew');
|
||||
genericGrab('http://chaosinthekitchen.com/2009/07/lime-and-coconut-chicken/');
|
||||
Loading…
Reference in New Issue
Block a user