martin/changedetection
martin/changedetection
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

added a timer based on ms

Browse Source
This commit is contained in:
Martin Donnelly 2019-10-29 12:25:06 +00:00
parent c869723a95
commit 63a9a11b79
3 changed files with 75 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.eslintrc
View File

@ -1,6 +1,6 @@
{ {
"parserOptions": { "parserOptions": {
"ecmaVersion": 2017, "ecmaVersion": 2018,
"sourceType": "module", "sourceType": "module",
"ecmaFeatures": { "ecmaFeatures": {
"jsx": false "jsx": false

21
changedetection.js
View File

@ -11,14 +11,29 @@ async function run() {
if (typeof(process.env.CD_CRON) === 'string' ) { if (typeof(process.env.CD_CRON) === 'string' ) {
console.log(`${cdScraper.id} cron set for ${process.env.CD_CRON}`); console.log(`${cdScraper.id} cron set for ${process.env.CD_CRON}`);
new CronJob(process.env.CD_CRON, async function() { await cdScraper.run();
new CronJob(process.env.CD_CRON, async () => {
console.log('go'); console.log('go');
await cdScraper.run(); await cdScraper.run();
console.log('ready to go again..');
}, null, true); }, null, true);
} }
if (process.env.SCRAPE_START === cdScraper.id) if (typeof(process.env.CD_EVERY) === 'string' ) {
{ console.log(`${cdScraper.id}_EVERY set for ${process.env.CD_EVERY} seconds`);
console.log('++');
await cdScraper.run();
console.log('--');
setInterval(async () => {
console.log('go');
await cdScraper.run();
console.log('ready to go again..');
// }, parseInt(process.env.CD_EVERY, 10));
}, process.env.CD_EVERY);
}
if (process.env.SCRAPE_START === cdScraper.id) {
console.log('go'); console.log('go');
await cdScraper.run(); await cdScraper.run();
} }

121
scrapers/scraper.js
View File

@ -10,13 +10,12 @@ const jsonfile = require('jsonfile');
const Diff = require('text-diff'); const Diff = require('text-diff');
const time = require("time-since"); const time = require('time-since');
const pug = require('pug'); const pug = require('pug');
const PNG = require('pngjs').PNG; const PNG = require('pngjs').PNG;
const pixelmatch = require('pixelmatch'); const pixelmatch = require('pixelmatch');
const compareImages = require("resemblejs/compareImages"); const compareImages = require('resemblejs/compareImages');
const email = require('smtp-email-sender')({ const email = require('smtp-email-sender')({
'host': 'mail.caliban.io', 'host': 'mail.caliban.io',
@ -38,21 +37,25 @@ class ChangeDetection extends Scraper {
this.setID('CD'); this.setID('CD');
this.run = this._debounce(async () => { /* this.run = _.debounce(async () => {
await this.__run(); await this.__run();
}, 5000); }, 5000);*/
this.run = async () => {
await this.__run();
};
} }
pugTest(data, newpath) { pugTest(data, newpath) {
logger.debug(pug.renderFile(`${newpath}/` + 'pug/email.pug', data)); logger.debug(pug.renderFile(`${newpath}/` + 'pug/email.pug', data));
} }
async sendSMTP(data, newPath) { async sendSMTP(data, newPath) {
const now = new Date(); const now = new Date();
const attachments = [ const attachments = [
{ {
path:data.diffPNG 'path':data.diffPNG
} }
]; ];
@ -62,86 +65,74 @@ class ChangeDetection extends Scraper {
'to': 'Martin <martind2000@gmail.com>', 'to': 'Martin <martind2000@gmail.com>',
'subject': `ChangeDetection: ${data.name}`, 'subject': `ChangeDetection: ${data.name}`,
'html': html, 'html': html,
attachments: attachments 'attachments': attachments
}); });
} }
async generateDiffScreenshotV2(previous, today) { async generateDiffScreenshotV2(previous, today) {
let {dir, root, ext, name} = path.parse(today); const { dir, root, ext } = path.parse(today);
let { name } = path.parse(today);
const options = { const options = {
output: { 'output': {
errorColor: { 'errorColor': {
red: 255, 'red': 255,
green: 0, 'green': 0,
blue: 255 'blue': 255
}, },
errorType: "movement", 'errorType': 'movement',
transparency: 0.3, 'transparency': 0.3,
largeImageThreshold: 1200, 'largeImageThreshold': 1200,
useCrossOrigin: false, 'useCrossOrigin': false,
outputDiff: true 'outputDiff': true
}, },
scaleToSameSize: false, 'scaleToSameSize': false,
ignore: "colors" 'ignore': 'colors'
}; };
// const img1 = PNG.sync.read(fs.readFileSync(previous)); // const img1 = PNG.sync.read(fs.readFileSync(previous));
// const img2 = PNG.sync.read(fs.readFileSync(today)); // const img2 = PNG.sync.read(fs.readFileSync(today));
const img1 = fs.readFileSync(previous); const img1 = fs.readFileSync(previous);
const img2 = fs.readFileSync(today); const img2 = fs.readFileSync(today);
const data = await compareImages(img1, img2, const data = await compareImages(img1, img2,
options options
); );
name = name.concat('_diff'); name = name.concat('_diff');
const endFilename = path.format({ dir, root, ext, name });
const endFilename = path.format({dir, root, ext, name}); logger.debug('diffFilename', endFilename);
fs.writeFileSync(endFilename, data.getBuffer());
return endFilename;
logger.debug('diffFilename', endFilename);
fs.writeFileSync(endFilename, data.getBuffer());
return endFilename;
} }
async generateDiffScreenshot(previous, today) { async generateDiffScreenshot(previous, today) {
const { dir, root, ext } = path.parse(today);
let {dir, root, ext, name} = path.parse(today); let { name } = path.parse(today);
const img1 = PNG.sync.read(fs.readFileSync(previous)); const img1 = PNG.sync.read(fs.readFileSync(previous));
const img2 = PNG.sync.read(fs.readFileSync(today)); const img2 = PNG.sync.read(fs.readFileSync(today));
const {width, height} = img1; const { width, height } = img1;
const diff = new PNG({width, height}); const diff = new PNG({ width, height });
pixelmatch(img1.data, img2.data, diff.data, width, height, {threshold: 0.1});
pixelmatch(img1.data, img2.data, diff.data, width, height, { 'threshold': 0.1 });
name = name.concat('_diff'); name = name.concat('_diff');
const endFilename = path.format({ dir, root, ext, name });
const endFilename = path.format({dir, root, ext, name});
logger.debug('diffFilename', endFilename); logger.debug('diffFilename', endFilename);
fs.writeFileSync(endFilename, PNG.sync.write(diff)); fs.writeFileSync(endFilename, PNG.sync.write(diff));
return endFilename; return endFilename;
} }
async processItem(item) { async processItem(item) {
logger.debug(`Processing ${item.name}...`); logger.debug(`Processing ${item.name}...`);
@ -178,9 +169,11 @@ class ChangeDetection extends Scraper {
const cleanedDiff = diff.cleanupSemantic(textDiff); const cleanedDiff = diff.cleanupSemantic(textDiff);
const levenshtein = diff.levenshtein(textDiff); const levenshtein = diff.levenshtein(textDiff);
// logger.debug('textDiff:', textDiff);
logger.debug('levenshtein:', levenshtein); logger.debug('levenshtein:', levenshtein);
logger.debug('cleanedDiff:',cleanedDiff ); logger.debug('cleanedDiff:', cleanedDiff );
if (levenshtein !== 0) { if (levenshtein !== 0) {
logger.info('Changed...'); logger.info('Changed...');
@ -203,13 +196,13 @@ class ChangeDetection extends Scraper {
fs.writeFileSync(oldFile, innerText.body, 'utf-8'); fs.writeFileSync(oldFile, innerText.body, 'utf-8');
this.stats.set(filename, stats); this.stats.set(filename, stats);
const pugData = {...stats, ...item}; const pugData = { ...stats, ...item}; // eslint-disable-line
// console.log(pugData); // console.log(pugData);
await this.sendSMTP(pugData, './'); await this.sendSMTP(pugData, './');
} else {
logger.debug('No change...');
} }
else
logger.debug('No change...');
} }
} }
@ -230,15 +223,12 @@ class ChangeDetection extends Scraper {
const oldFile = `${this.path}/previous.html`; const oldFile = `${this.path}/previous.html`;
const innerText = await this.page.evaluate(() => { const innerText = await this.page.evaluate(() => {
return { return {
'body': document.body.innerText 'body': document.body.innerText
}; };
}); });
if (!fs.existsSync(oldFile)) if (!fs.existsSync(oldFile))
fs.writeFileSync(oldFile, body.body, 'utf-8'); fs.writeFileSync(oldFile, body.body, 'utf-8');
else { else {
@ -253,7 +243,7 @@ class ChangeDetection extends Scraper {
if (levenshtein !== 0) { if (levenshtein !== 0) {
logger.debug(diff.prettyHtml(textDiff)); logger.debug(diff.prettyHtml(textDiff));
fs.writeFileSync(oldFile, innerText.body, 'utf-8'); fs.writeFileSync(oldFile, innerText.body, 'utf-8');
} }
} }
} }
@ -316,7 +306,7 @@ class ChangeDetection extends Scraper {
* @returns {Promise<void>} * @returns {Promise<void>}
*/ */
async __run() { async __run() {
// try { // try {
logger.debug('run'); logger.debug('run');
await this.start(); await this.start();
@ -330,8 +320,9 @@ class ChangeDetection extends Scraper {
await this.saveSettings(); await this.saveSettings();
await this._done(); await this._done();
/*}
/* }
catch(e) { catch(e) {
throw new Error(e); throw new Error(e);
}*/ }*/