794 lines
17 KiB
JavaScript
794 lines
17 KiB
JavaScript
const fs = require('fs-extra');
|
|
const path = require('path');
|
|
const url = require('url');
|
|
const log4js = require('log4js');
|
|
let logger = log4js.getLogger('Scraper');
|
|
const del = require('del');
|
|
const EventEmitter = require('events');
|
|
const dateFormat = require('dateformat');
|
|
|
|
const puppeteer = require('puppeteer');
|
|
|
|
logger.level = process.env.LOGGER_LEVEL || 'debug';
|
|
|
|
class Scraper extends EventEmitter {
|
|
|
|
constructor() {
|
|
super(); // must call super for "this" to be defined.
|
|
|
|
this.filters = [
|
|
'livefyre',
|
|
'moatad',
|
|
'analytics',
|
|
'controltag',
|
|
'chartbeat',
|
|
'siteimprove',
|
|
'hotjar',
|
|
'/plugins/cookie-notice/',
|
|
'addthis',
|
|
'facebook.',
|
|
'linkedin',
|
|
'googletagmanager',
|
|
'swiftypecdn.com',
|
|
'-social-tracking.',
|
|
'demdex.net',
|
|
'adobedtm.com'
|
|
];
|
|
|
|
this.perf = {
|
|
'started': 0,
|
|
'finished': 0,
|
|
'time': 0,
|
|
'scraped': 0
|
|
};
|
|
|
|
this.browserCrashed = false;
|
|
this.crashLog = new Map([]);
|
|
|
|
this.page = null;
|
|
}
|
|
|
|
setID(newID) {
|
|
logger = log4js.getLogger(`Scraper (${newID})`);
|
|
logger.level = process.env.LOGGER_LEVEL || 'warn';
|
|
|
|
this.id = newID;
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param path
|
|
* @returns {Promise<void>}
|
|
*/
|
|
|
|
async emptyPath(path) {
|
|
if (process.env.NODE_ENV === 'production')
|
|
await del([path]).then(paths => {
|
|
logger.warn('Deleted files and folders:\n', paths.join('\n'));
|
|
});
|
|
}
|
|
|
|
async setPath(newPath) {
|
|
const now = new Date();
|
|
const timestamp = dateFormat(now, 'yyyymmdd');
|
|
|
|
// why delete??
|
|
// await this.emptyPath(newPath);
|
|
|
|
// this.path = `${newPath}/${timestamp}`;
|
|
this.path = `${newPath}`;
|
|
this.debugPath = `${__dirname }/../debug/${this.id}`;
|
|
await this._createDirectory(this.path);
|
|
await this._createDirectory(this.debugPath);
|
|
}
|
|
|
|
/**
|
|
* 'Human' like click delay
|
|
* @returns {number}
|
|
*/
|
|
static notARobot() {
|
|
return 90 + Math.floor(Math.random() * (30 - 1));
|
|
}
|
|
|
|
/**
|
|
*
|
|
*/
|
|
canDetach() {
|
|
this.detatchable = true;
|
|
}
|
|
|
|
async _killRunningBrowser() {
|
|
// if (typeof(this.browser) !== 'undefined' && this.browser !== null) {
|
|
if (this.browser)
|
|
try{
|
|
logger.info('Trying to close hanging / running browser');
|
|
|
|
await this._forcePageClose();
|
|
|
|
await this.browser.removeAllListeners('disconnected');
|
|
|
|
await this.browser.close();
|
|
}
|
|
catch(err) {
|
|
logger.error('Closing browser', err);
|
|
}
|
|
finally {
|
|
this.browser = null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param headless
|
|
* @returns {Promise<void>}
|
|
* @private
|
|
*/
|
|
async _initBrowser(headless = true) {
|
|
// Force headless when running in production
|
|
|
|
const realHeadless = (process.env.NODE_ENV === 'production') ? true : headless;
|
|
|
|
await this._killRunningBrowser();
|
|
|
|
this.browserCrashed = false;
|
|
|
|
logger.info('Puppeteer.launch', realHeadless);
|
|
|
|
logger.debug('Using proxy:', process.env.PROXY_URI);
|
|
this.browser = await puppeteer.launch({
|
|
'headless': realHeadless,
|
|
'args': [
|
|
// Use proxy so FCA wont block us
|
|
`--proxy-server=${process.env.PROXY_URI}`,
|
|
'--disable-dev-shm-usage',
|
|
'--no-sandbox',
|
|
'--disable-setuid-sandbox',
|
|
'--disable-accelerated-2d-canvas',
|
|
'--disable-gpu',
|
|
'--window-size=1920x1080',
|
|
'--hide-scrollbars',
|
|
'--disable-default-apps'
|
|
]
|
|
}).catch((err) => {
|
|
logger.error('Puppeteer failed to launch');
|
|
logger.error(err);
|
|
});
|
|
|
|
const browserVersion = await this.browser.version();
|
|
|
|
logger.info(`Browser version ${browserVersion}`);
|
|
|
|
this.browser.on('disconnected', () => {
|
|
logger.warn('Browser has become detached!');
|
|
|
|
if (this.detatchable === false) {
|
|
this.browserCrashed = true;
|
|
|
|
logger.warn('browser.onDisconnected::emit recover');
|
|
this.emit('recover');
|
|
}
|
|
});
|
|
}
|
|
|
|
async _forcePageClose() {
|
|
// if (this.page !== null) {
|
|
if (this.page)
|
|
|
|
try{
|
|
logger.warn('Browser Page exists: DESTROYING');
|
|
|
|
await this.page.removeAllListeners('close');
|
|
// this.page.on('close', () => {});
|
|
|
|
await this.page.close().catch((e) => {
|
|
logger.debug(e);
|
|
});
|
|
}
|
|
catch( err) {
|
|
logger.error(err);
|
|
}
|
|
finally {
|
|
this.page = null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @returns {Promise<void>}
|
|
* @private
|
|
*/
|
|
async _createBrowserPage() {
|
|
this._forcePageClose();
|
|
|
|
this.page = await this.browser.newPage();
|
|
|
|
try{
|
|
await this.page.setDefaultNavigationTimeout(90000);
|
|
|
|
await this.page.setDefaultTimeout(90000);
|
|
}
|
|
catch(err) {
|
|
logger.debug(err);
|
|
}
|
|
|
|
await this.page.setRequestInterception(true);
|
|
|
|
this.page.on('request', (request) => {
|
|
const url = request.url();
|
|
logger.trace('request', url);
|
|
const shouldAbort = this.filters.some((urlPart) => url.includes(urlPart));
|
|
if (shouldAbort) request.abort();
|
|
else request.continue();
|
|
});
|
|
|
|
this.page.on('dialog', async dialog => {
|
|
logger.warn('Dialog Box', dialog.message());
|
|
await dialog.dismiss();
|
|
});
|
|
|
|
this.page.on('error', async err => {
|
|
logger.warn('Page crashed', err);
|
|
if (!this.detatchable) {
|
|
await this._uploadError();
|
|
logger.warn('page.onError::emit recover');
|
|
this.emit('recover');
|
|
}
|
|
});
|
|
|
|
this.page.on('pageerror', async err => {
|
|
logger.trace('pageerror', err);
|
|
});
|
|
|
|
this.page.on('requestfailed', async err => {
|
|
const url = err['_url'];
|
|
const blocked = this.filters.some((urlPart) => url.includes(urlPart));
|
|
|
|
if (blocked)
|
|
logger.trace('🚫', err['_url']);
|
|
else
|
|
logger.warn('requestfailed', err['_url']);
|
|
});
|
|
|
|
this.page.on('close', () => {
|
|
logger.warn('Browser Page has closed');
|
|
|
|
if (this.detatchable === false) {
|
|
logger.warn('page.onClose::emit recover');
|
|
this.emit('recover');
|
|
}
|
|
});
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @returns {Promise<void>}
|
|
* @private
|
|
*/
|
|
async _makeResponsive() {
|
|
const viewPort = {
|
|
'name': 'Responsive',
|
|
'userAgent' : 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3494.0 Safari/537.36',
|
|
'viewport': {
|
|
'width': 1200,
|
|
'height': 1200,
|
|
'deviceScaleFactor': 4.5,
|
|
'isMobile': true,
|
|
'hasTouch': true,
|
|
'isLandscape': true
|
|
}
|
|
};
|
|
|
|
await this.page.setViewport(viewPort.viewport);
|
|
|
|
await this.page.setDefaultNavigationTimeout(90000);
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param id
|
|
* @returns {string}
|
|
* @private
|
|
*/
|
|
_makeFileName(id) {
|
|
const noWhiteSpace = /\W/g;
|
|
const maxChars = 175;
|
|
const entity = removeAccents.remove(id.replace(noWhiteSpace, ' ').trim());
|
|
|
|
const _crc = crc.crc32(id).toString(16);
|
|
|
|
const output = [this.modePrefix[this.mode], camelCase(entity)].join('');
|
|
|
|
return (output.length > maxChars) ? output.substring(0, maxChars).concat('_', _crc) : output;
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param id
|
|
* @returns {Promise<string>}
|
|
* @private
|
|
*/
|
|
async _makeFilePath(id) {
|
|
return `${this.path}/${this._makeFileName(id)}`.substring(0, 240);
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param page
|
|
* @param destPath
|
|
* @param waitFor
|
|
* @returns {Promise<void>}
|
|
* @private
|
|
*/
|
|
async _makeScreenshotV2(page, destPath, waitFor = null) {
|
|
try{
|
|
if (waitFor)
|
|
await page.waitFor(waitFor);
|
|
|
|
if(!this.page) {
|
|
logger.warn('_makeScreenshotV2: No Page -- Not taking screenshot');
|
|
|
|
return;
|
|
}
|
|
|
|
logger.debug('Snapshot', `${destPath}.png`);
|
|
await page.setViewport({ 'width': 1200, 'height': 800 });
|
|
await page.screenshot({ 'path': `${destPath}.png`, 'fullPage': true }).catch(err => {
|
|
logger.error('Screenshot', err);
|
|
});
|
|
}
|
|
catch( err) {
|
|
logger.error('_makeScreenshotV2', err);
|
|
}
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param page
|
|
* @param minTime
|
|
* @param maxTime
|
|
* @param msg
|
|
* @returns {Promise<void>}
|
|
* @private
|
|
*/
|
|
async _randomWait(page, minTime = 2, maxTime = 10, msg = '') {
|
|
const insertedMsg = (msg.length > 0) ? `${this.id} ${msg} - ` : `${this.id} `;
|
|
|
|
const waitTime = Math.floor(Math.random() * (maxTime - minTime + 1) + minTime);
|
|
logger.debug(`${insertedMsg}Waiting ${waitTime} seconds...`);
|
|
await page.waitFor(waitTime * 1000);
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param page
|
|
* @param waitTime
|
|
* @param msg
|
|
* @returns {Promise<void>}
|
|
* @private
|
|
*/
|
|
async _microWait(page, waitTime, msg = '') {
|
|
const insertedMsg = (msg.length > 0) ? `${msg} - ` : '';
|
|
|
|
if (msg !== '') logger.debug(`${insertedMsg}Waiting ${waitTime * 100} ms...`);
|
|
await page.waitFor(waitTime * 100);
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param page
|
|
* @param waitTime
|
|
* @param msg
|
|
* @returns {Promise<void>}
|
|
* @private
|
|
*/
|
|
async _nanoWait(page, waitTime, msg = '') {
|
|
const insertedMsg = (msg.length > 0) ? `${msg} - ` : '';
|
|
|
|
if (msg !== '') logger.debug(`${insertedMsg}Waiting ${waitTime * 10} ms...`);
|
|
await page.waitFor(waitTime * 10);
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param destPath
|
|
* @param data
|
|
* @returns {Promise<*>}
|
|
* @private
|
|
*/
|
|
async _saveToFile(destPath, data) {
|
|
// use for artefacts saving only
|
|
return new Promise((resolve, reject) => {
|
|
const fullPath = `${__dirname}/../artefacts/${destPath}`;
|
|
fs.writeFile(fullPath, data, function(err) {
|
|
if(err)
|
|
reject(err);
|
|
else
|
|
resolve(`File saved to '${fullPath}'`);
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param destPath
|
|
* @param data
|
|
* @returns {Promise<*>}
|
|
* @private
|
|
*/
|
|
async _dumpFile(destPath, data) {
|
|
return new Promise((resolve, reject) => {
|
|
fs.writeFile(destPath, data, function(err) {
|
|
if(err)
|
|
reject(err);
|
|
else
|
|
resolve(`File saved to '${destPath}'`);
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param destPath
|
|
* @returns {Promise<string>}
|
|
* @private
|
|
*/
|
|
async _createTimestampDirectory(destPath = null) {
|
|
const now = new Date();
|
|
|
|
const timestamp = dateFormat(now, 'yyyymmddHHMM');
|
|
const fullPath = `${destPath}/${timestamp}`;
|
|
|
|
logger.info('fullPath', fullPath);
|
|
|
|
if (!fs.existsSync(fullPath))
|
|
fs.ensureDirSync(fullPath);
|
|
|
|
return fullPath;
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param destPath
|
|
* @returns {Promise<*>}
|
|
* @private
|
|
*/
|
|
async _createDirectory(destPath = null) {
|
|
try{
|
|
if (!fs.existsSync(destPath))
|
|
fs.ensureDirSync(destPath);
|
|
}
|
|
catch( err) {
|
|
logger.error('_createDirectory', err);
|
|
}
|
|
|
|
return destPath;
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param destPath
|
|
* @param filename
|
|
* @returns {Promise<*>}
|
|
* @private
|
|
*/
|
|
async _createArchive(destPath = null, filename = null, glob = false) {
|
|
return new Promise((resolve, reject) => {
|
|
if (!destPath || !filename) {
|
|
const e = new Error('Missing paths');
|
|
logger.error(e);
|
|
reject(e);
|
|
}
|
|
const archive = archiver(filename, {
|
|
'zlib': { 'level': 9 } // Sets the compression level.
|
|
});
|
|
|
|
if (glob)
|
|
archive.glob(`${destPath}`);
|
|
else
|
|
archive.directory(`${destPath}/`);
|
|
|
|
archive.finalize().then(() => {
|
|
logger.debug('Archive finished');
|
|
resolve();
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param destPath
|
|
* @param filename
|
|
* @param glob
|
|
* @returns {Promise<*>}
|
|
* @private
|
|
*/
|
|
async _createArchiveV2(destPath = null, filename = null, glob = false) {
|
|
logger.debug('=== _createArchiveV2 :: STREAMING ===');
|
|
|
|
return new Promise((resolve, reject) => {
|
|
if (!destPath || !filename) {
|
|
const e = new Error('Missing paths');
|
|
logger.error(e);
|
|
reject(e);
|
|
}
|
|
|
|
const output = fs.createWriteStream(filename);
|
|
|
|
const archive = archiver('zip', {
|
|
'TransformOptions': {
|
|
'objectMode':true
|
|
},
|
|
'zlib': { 'level': 6 } // Sets the compression level.
|
|
});
|
|
|
|
archive.pipe(output);
|
|
|
|
if (glob)
|
|
archive.glob(`${destPath}`);
|
|
else
|
|
archive.directory(`${destPath}/`);
|
|
|
|
archive.finalize().then(() => {
|
|
logger.debug('Archive finished');
|
|
resolve();
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param urlStr
|
|
* @returns {*}
|
|
*/
|
|
explodeURL (urlStr = null) {
|
|
if (!urlStr || urlStr === '')
|
|
return (null);
|
|
|
|
try {
|
|
const workURL = url.parse(urlStr);
|
|
|
|
return tldExtract.parse_host( workURL.host);
|
|
}
|
|
catch(e) {
|
|
return e;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get Params from a url string
|
|
*/
|
|
_getParamsFromUrl(url) {
|
|
url = decodeURI(url);
|
|
if (typeof url === 'string') {
|
|
const params = url.split('?');
|
|
|
|
const obj = {};
|
|
if (params.length > 1) {
|
|
const eachParamsArr = params[1].split('&');
|
|
|
|
if (eachParamsArr && eachParamsArr.length)
|
|
eachParamsArr.map(param => {
|
|
const keyValuePair = param.split('=');
|
|
const key = keyValuePair[0];
|
|
const value = keyValuePair[1];
|
|
obj[key] = value;
|
|
});
|
|
}
|
|
|
|
return obj;
|
|
}
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param text
|
|
* @returns {string}
|
|
* @private
|
|
*/
|
|
_cleanUp(text) {
|
|
if (!text) return '';
|
|
const regexNewLine = /\n/;
|
|
const regexCollapseWS = /\s+/g;
|
|
|
|
return text.replace(regexNewLine, '').replace(regexCollapseWS, ' ').trim();
|
|
}
|
|
|
|
_makeFieldName(text) {
|
|
const removePunctuation = /([^A-Za-z0-9\s])+/g;
|
|
|
|
if (!text) return '';
|
|
let workString = this._cleanUp(text);
|
|
workString = removeAccents.remove(workString);
|
|
workString = workString.replace(removePunctuation, '');
|
|
|
|
workString = camelCase(workString);
|
|
|
|
return workString;
|
|
}
|
|
|
|
async _renameFile(origFN, newFN) {
|
|
await checkFileExists(origFN)
|
|
.then(async exists => {
|
|
console.log(`file exists: ${exists}`);
|
|
|
|
if (exists)
|
|
await fs.renameSync(origFN, newFN);
|
|
}).catch((e) => {
|
|
logger.error(e);
|
|
});
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @private
|
|
*/
|
|
async _start() {
|
|
logger.debug(`<=- START ${this.id}-=>`);
|
|
const now = new Date();
|
|
this.perf.started = now.getTime();
|
|
|
|
this.on('recover', async () => {
|
|
await this.recover();
|
|
});
|
|
|
|
// await this._createLock();
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @returns {Promise<void>}
|
|
* @private
|
|
*/
|
|
async _done() {
|
|
logger.info('<=- DONE -=>');
|
|
|
|
// OK To close the browser window now
|
|
this.canDetach();
|
|
|
|
await this._forcePageClose();
|
|
|
|
await this._killRunningBrowser();
|
|
|
|
await this._complete();
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @returns {Promise<void>}
|
|
* @private
|
|
*/
|
|
async _complete() {
|
|
try {
|
|
if (global.gc) global.gc();
|
|
}
|
|
catch (e) {
|
|
logger.warn('`node --expose-gc`');
|
|
}
|
|
|
|
logger.info('<=- COMPLETE -=>');
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param url
|
|
* @param options
|
|
* @param noRecover
|
|
* @returns {Promise<void>}
|
|
* @private
|
|
*/
|
|
async _goto(url, options = {}, noRecover = false) {
|
|
this.lastUrl = url;
|
|
|
|
const newOptions = Object.assign({ 'timeout':90000, 'waitUntil':'networkidle0' }, options);
|
|
|
|
logger.debug(newOptions);
|
|
|
|
try {
|
|
logger.info('Goto:', url);
|
|
await this.page.goto(url, newOptions).catch((err) => {
|
|
logger.error('GOTO', err);
|
|
|
|
if (err.message.indexOf('net::ERR_FAILED') !== -1)
|
|
this.browserCrashed = true;
|
|
|
|
if (!noRecover)
|
|
this.emit('recover');
|
|
});
|
|
}
|
|
catch (error) {
|
|
logger.error(error);
|
|
logger.error(url, options);
|
|
// if (error === 'net::ERR_CONNECTION_TIMED_OUT')
|
|
}
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param fn
|
|
* @param time
|
|
* @returns {Function}
|
|
* @private
|
|
*/
|
|
_debounce(fn, time) {
|
|
let timeout;
|
|
|
|
return function (...args) { // <-- not an arrow function
|
|
const functionCall = () => fn.apply(this, args);
|
|
|
|
clearTimeout(timeout);
|
|
timeout = setTimeout(functionCall, time);
|
|
};
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param callback
|
|
* @param limit
|
|
* @returns {Function}
|
|
* @private
|
|
*/
|
|
_throttle (callback, limit) {
|
|
var wait = false;
|
|
|
|
return function () {
|
|
if (!wait) {
|
|
callback.apply(null, arguments);
|
|
wait = true;
|
|
setTimeout(function () {
|
|
wait = false;
|
|
}, limit);
|
|
}
|
|
};
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param func
|
|
* @returns {function(): *}
|
|
* @private
|
|
*/
|
|
|
|
_once(func) {
|
|
var alreadyCalled = false;
|
|
var result;
|
|
|
|
return function() {
|
|
if (!alreadyCalled) {
|
|
result = func.apply(this, arguments);
|
|
alreadyCalled = true;
|
|
}
|
|
|
|
return result;
|
|
};
|
|
};
|
|
|
|
/**
|
|
*
|
|
* @param restartURL
|
|
* @returns {Promise<void>}
|
|
*/
|
|
async restart(restartURL) {
|
|
const rURL = restartURL || this.lastUrl;
|
|
logger.info(`Restarting ${this.id} // Going to ${rURL}`);
|
|
|
|
await this._goto(rURL);
|
|
}
|
|
|
|
/**
|
|
*
|
|
* @param filename
|
|
* @param data
|
|
* @returns {Promise<void>}
|
|
*/
|
|
async saveFile(filename, data) {
|
|
try{
|
|
fs.writeFileSync(filename, data);
|
|
}
|
|
catch( err) {
|
|
logger.error(err);
|
|
}
|
|
}
|
|
}
|
|
|
|
module.exports = Scraper;
|