diff --git a/.gitignore b/.gitignore index b147590..7fb3689 100644 --- a/.gitignore +++ b/.gitignore @@ -159,3 +159,4 @@ artefacts/* /archive.tar.gz /user/ +/zip diff --git a/Jenkinsfile b/Jenkinsfile index 3c20e2b..0d1dcf7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,6 +1,3 @@ -@Library('OpenBankingUK/ob-pipeline-library') _ +@Library('OpenBankingUK/devdir-jenkins-pipeline-lib') _ -javaMsPipelinev2 { - projectName='obdfcascrape' - cluster='nca' -} +buildNodeMsPipeline {projectName='obdfcascrape'} diff --git a/ecosystem.config.js b/ecosystem.config.js index 3a8b993..9830de4 100644 --- a/ecosystem.config.js +++ b/ecosystem.config.js @@ -25,14 +25,14 @@ function buildApps() { { 'cron':'DE_CRON', 'start':'DE', 'name':'DE', 'script':'de.js', 'proxy': 'de', 'crontime': '0 0 * * *' }, // 03:55:38 - de free at 4:00 { 'cron':'NL_CRON', 'start':'NL', 'name':'NL', 'script':'nl.js', 'proxy': 'nl', 'crontime': '0 0 * * *' }, // 07:23:19 - nl free at 7:30 { 'cron':'PL_CRON', 'start':'PL', 'name':'PL', 'script':'pl.js', 'proxy': 'ch', 'crontime': '0 0 * * *' }, // 17:59:18 - ch free at 18:00 - { 'cron':'DK_CRON', 'start':'DK', 'name':'DK', 'script':'dk.js', 'proxy': 'uk' }, - { 'cron':'ES_CRON', 'start':'ES', 'name':'ES', 'script':'es.js', 'proxy': 'uk' }, + { 'cron':'LV_CRON', 'start':'LV', 'name':'LV', 'script':'lv.js', 'proxy': 'nl', 'crontime': '30 7 * * *' }, // 13:56.232 - nl free at 7:45 + { 'cron':'DK_CRON', 'start':'DK', 'name':'DK', 'script':'dk.js', 'proxy': 'de', 'crontime': '0 4 * * *' }, // 11:08.616 - de free at 4:15 + { 'cron':'ES_CRON', 'start':'ES', 'name':'ES', 'script':'es.js', 'proxy': 'de', 'crontime': '15 4 * * *' }, // 36:44.523- de free at 4:55 + { 'cron':'EE_CRON', 'start':'EE', 'name':'EE', 'script':'ee.js', 'proxy': 'de', 'crontime': '0 5 * * *' }, // 05:22:04.226 - de free after 10:30 + { 'cron':'NO_CRON', 'start':'NO', 'name':'NO', 'script':'no.js', 'proxy': 'fr', 'crontime': '0 4 * * *' }, // 05:12:57.792 - fr free after 9:20 { 'cron':'GI_CRON', 'start':'GI', 'name':'GI', 'script':'gi.js', 'proxy': 'uk' }, { 'cron':'GR_CRON', 'start':'GR', 'name':'GR', 'script':'gr.js', 'proxy': 'uk' }, { 'cron':'MT_CRON', 'start':'MT', 'name':'MT', 'script':'mt.js', 'proxy': 'uk' }, - { 'cron':'LV_CRON', 'start':'LV', 'name':'LV', 'script':'lv.js', 'proxy': 'uk' }, - { 'cron':'NO_CRON', 'start':'NO', 'name':'NO', 'script':'no.js', 'proxy': 'uk' }, - { 'cron':'EE_CRON', 'start':'EE', 'name':'EE', 'script':'ee.js', 'proxy': 'uk' }, { 'cron':'BG_CRON', 'start':'BG', 'name':'BG', 'script':'bg.js', 'proxy': 'uk' }, { 'cron':'AT_CRON', 'start':'AT', 'name':'AT', 'script':'at.js', 'proxy': 'uk' }, { 'cron':'FI_CRON', 'start':'FI', 'name':'FI', 'script':'fi.js', 'proxy': 'uk' }, diff --git a/helpers/scraper.js b/helpers/scraper.js index a7e8eed..249ca1e 100644 --- a/helpers/scraper.js +++ b/helpers/scraper.js @@ -52,7 +52,7 @@ const useDig = ['cy']; // Default the region AWS.config.update({ 'region': 'eu-west-1' }); -if (process.env.NODE_ENV !== 'production') +if (process.env.NODE_ENV !== 'production') AWS.config.update({ 'accessKeyId': process.env.AWS_ACCESS_KEY_ID, 'secretAccessKey': process.env.AWS_SECRET_ACCESS_KEY, 'region': process.env.AWS_REGION || 'eu-west-1' }); const s3 = new AWS.S3(); @@ -143,7 +143,7 @@ class Scraper extends EventEmitter { */ async emptyPath(path) { - if (process.env.NODE_ENV === 'production') + if (process.env.NODE_ENV === 'production') await del([path]).then(paths => { logger.warn('Deleted files and folders:\n', paths.join('\n')); }); @@ -165,7 +165,7 @@ class Scraper extends EventEmitter { await this._createDirectory(this.path); await this._createDirectory(this.debugPath); } - + /** * 'Human' like click delay * @returns {number} @@ -183,7 +183,7 @@ class Scraper extends EventEmitter { async _killRunningBrowser() { // if (typeof(this.browser) !== 'undefined' && this.browser !== null) { - if (this.browser) + if (this.browser) try{ logger.info('Trying to close hanging / running browser'); @@ -231,14 +231,17 @@ class Scraper extends EventEmitter { '--disable-gpu', '--window-size=1920x1080', '--hide-scrollbars', - '--disable-default-apps', - '--remote-debugging-port=9222' + '--disable-default-apps' ] }).catch((err) => { logger.error('Puppeteer failed to launch'); logger.error(err); }); + const browserVersion = await this.browser.version(); + + logger.info(`Browser version ${browserVersion}`); + this.browser.on('disconnected', () => { logger.warn('Browser has become detached!'); @@ -253,7 +256,7 @@ class Scraper extends EventEmitter { async _forcePageClose() { // if (this.page !== null) { - if (this.page) + if (this.page) try{ logger.warn('Browser Page exists: DESTROYING'); @@ -309,9 +312,11 @@ class Scraper extends EventEmitter { this.page.on('error', async err => { logger.warn('Page crashed', err); - await this._uploadError(); - logger.warn('page.onError::emit recover'); - this.emit('recover'); + if (!this.detatchable) { + await this._uploadError(); + logger.warn('page.onError::emit recover'); + this.emit('recover'); + } }); this.page.on('pageerror', async err => { @@ -399,9 +404,9 @@ class Scraper extends EventEmitter { * @private */ async _makeScreenshot(page, destPath, waitFor = null) { - if (waitFor) + if (waitFor) await page.waitFor(waitFor); - + await page.setViewport({ 'width': 1200, 'height': 800 }); await page.screenshot({ 'path': `artefacts/screenshots/${destPath}.png`, 'fullPage': true }).catch((err) => { logger.error('Screenshot', err); @@ -449,7 +454,7 @@ class Scraper extends EventEmitter { */ async _randomWait(page, minTime = 2, maxTime = 10, msg = '') { const insertedMsg = (msg.length > 0) ? `${this.id} ${msg} - ` : `${this.id} `; - + const waitTime = Math.floor(Math.random() * (maxTime - minTime + 1) + minTime); logger.debug(`${insertedMsg}Waiting ${waitTime} seconds...`); await page.waitFor(waitTime * 1000); @@ -497,9 +502,9 @@ class Scraper extends EventEmitter { return new Promise((resolve, reject) => { const fullPath = `${__dirname}/../artefacts/${destPath}`; fs.writeFile(fullPath, data, function(err) { - if(err) + if(err) reject(err); - else + else resolve(`File saved to '${fullPath}'`); }); }); @@ -579,11 +584,11 @@ class Scraper extends EventEmitter { 'zlib': { 'level': 9 } // Sets the compression level. }); - if (glob) + if (glob) archive.glob(`${destPath}`); - else + else archive.directory(`${destPath}/`); - + archive.finalize().then(() => { logger.debug('Archive finished'); resolve(); @@ -595,6 +600,7 @@ class Scraper extends EventEmitter { * * @param destPath * @param filename + * @param glob * @returns {Promise<*>} * @private */ @@ -693,7 +699,7 @@ class Scraper extends EventEmitter { async _getWhoIsJSON(destPath = null, withPrefix = false) { const options = { }; - if (!destPath) + if (!destPath) throw new Error('No destination path'); const explodedURL = this.explodeURL(destPath); @@ -973,7 +979,7 @@ class Scraper extends EventEmitter { .then(async exists => { console.log(`file exists: ${exists}`); - if (exists) + if (exists) await fs.renameSync(origFN, newFN); }).catch((e) => { logger.error(e); @@ -1008,7 +1014,7 @@ class Scraper extends EventEmitter { await this._checkS3FileExists(langFileName) .then(exists => { - if (exists) + if (exists) return new Promise((resolve, reject) => { // (*) this._getFileS3(langFileName).then((data) => { this.dictionary = new Map(JSON.parse(data)); @@ -1030,7 +1036,7 @@ class Scraper extends EventEmitter { async _saveDictionary() { if (this.dictionary.size > 0) { logger.debug('Save dictionary', this.dictionary.size); - + return new Promise((resolve, reject) => { const langFileName = `lang.${this.id.toLowerCase()}.json`; const arrayedMap = JSON.stringify([...this.dictionary]); @@ -1099,6 +1105,9 @@ class Scraper extends EventEmitter { async _done() { logger.info('<=- DONE -=>'); + // OK To close the browser window now + this.canDetach(); + const now = new Date(); this.perf.finished = now.getTime(); @@ -1114,9 +1123,6 @@ class Scraper extends EventEmitter { await this._archive(); - // OK To close the browser window now - this.canDetach(); - await this._forcePageClose(); await this._killRunningBrowser(); @@ -1307,9 +1313,9 @@ class Scraper extends EventEmitter { }; return new Promise((resolve, reject) => { - if (filename === null) + if (filename === null) return reject(Error('No filename for S3')); - + s3.headObject(params).promise().then((i) => { logger.debug(`${filename} exists`); @@ -1366,7 +1372,7 @@ class Scraper extends EventEmitter { */ async _sendMessage(id, msg, msgBody = 'New upload') { logger.debug('+ _sendMessage', process.env.SQS_ID); - if (typeof process.env.SQS_ID !== 'undefined' && process.env.SQS_ID !== null) + if (typeof process.env.SQS_ID !== 'undefined' && process.env.SQS_ID !== null) try { const sqs = new AWS.SQS({ 'apiVersion': '2012-11-05' }); @@ -1394,7 +1400,7 @@ class Scraper extends EventEmitter { return data; }).catch((err) => { logger.error(err); - + return err; }); } @@ -1469,7 +1475,7 @@ class Scraper extends EventEmitter { * @private */ async _doNonRepudiation(skip = false, options = {}) { - if (!skip) + if (!skip) try{ if (typeof this.startPage === 'undefined' || this.startPage === null) @@ -1506,12 +1512,24 @@ class Scraper extends EventEmitter { const key = localStorage.key(i); json[key] = localStorage.getItem(key); } - + return json; }); await jsonfile.writeFileSync(filePath, json); } + async _getLocalStorage( ) { + return await this.page.evaluate(() => { + const json = {}; + for (let i = 0; i < localStorage.length; i++) { + const key = localStorage.key(i); + json[key] = localStorage.getItem(key); + } + + return json; + }); + } + _checkFileExistsSync(filePath) { try { fs.accessSync(filePath, fs.F_OK); @@ -1568,7 +1586,7 @@ class Scraper extends EventEmitter { if (err.message.indexOf('net::ERR_FAILED') !== -1) this.browserCrashed = true; - if (!noRecover) + if (!noRecover) this.emit('recover'); }); } @@ -1606,7 +1624,7 @@ class Scraper extends EventEmitter { */ _throttle (callback, limit) { var wait = false; - + return function () { if (!wait) { callback.apply(null, arguments); @@ -1628,13 +1646,13 @@ class Scraper extends EventEmitter { _once(func) { var alreadyCalled = false; var result; - + return function() { if (!alreadyCalled) { result = func.apply(this, arguments); alreadyCalled = true; } - + return result; }; }; @@ -1664,39 +1682,54 @@ class Scraper extends EventEmitter { async __recover(restartURL) { logger.warn(`*** RECONNECTING ${this.id} PAGE ***`); + let crashCount = 0; if (this.crashLog.has(this.lastUrl)) { - let crashCount = this.crashLog.get(this.lastUrl); + crashCount = this.crashLog.get(this.lastUrl); crashCount++; this.crashLog.set(this.lastUrl, crashCount); - if (crashCount >= 3) + if (crashCount >= 3) logger.error('The page has crashed more than 3 times', this.lastUrl); + + if (crashCount >= 10) { + logger.error('10 times on the same page is enough', this.lastUrl); + + return; + } } - else + else this.crashLog.set(this.lastUrl, 1); - if (this.browserCrashed) await this._initBrowser(true); + if (crashCount < 10) { + if (this.browserCrashed) await this._initBrowser(true); - await this._createBrowserPage(); + await this._createBrowserPage(); - logger.debug('Reattach processNewPage', (typeof this.processNewPage === 'function') ? 'Yes' : 'No'); - if (typeof this.processNewPage === 'function') - this.page.on('domcontentloaded', () => { - this.processNewPage(); - }); + logger.debug('Reattach processNewPage', (typeof this.processNewPage === 'function') ? 'Yes' : 'No'); + if (typeof this.processNewPage === 'function') + this.page.on('domcontentloaded', () => { + this.processNewPage(); + }); - const antiCollision = 125 + (Math.floor(Math.random() * (15 - 1)) * 500); - const timeout = 90000 + antiCollision; + const onHold = (crashCount >= 3) ? (90000 * crashCount) : 0; + const antiCollision = 125 + (Math.floor(Math.random() * (15 - 1)) * 500); + const timeout = 90000 + antiCollision + onHold; - logger.info(`🚨 Restarting in ${(timeout / 1000).toFixed(2)} seconds.`); + logger.info(`🚨 Restarting in ${(timeout / 1000).toFixed(2)} seconds.`); - setTimeout(async() => { - logger.warn('Attempting recovery..'); + setTimeout(async() => { + logger.warn(`Attempting recovery to ${restartURL}`); - await this.restart(restartURL); - }, timeout); + await this.restart(restartURL); + }, timeout); + } } + /** + * + * @param restartURL + * @returns {Promise} + */ async restart(restartURL) { const rURL = restartURL || this.lastUrl; logger.info(`Restarting ${this.id} // Going to ${rURL}`); @@ -1704,6 +1737,12 @@ class Scraper extends EventEmitter { await this._goto(rURL); } + /** + * + * @param filename + * @param data + * @returns {Promise} + */ async saveFile(filename, data) { try{ fs.writeFileSync(filename, data); @@ -1713,6 +1752,40 @@ class Scraper extends EventEmitter { } } + /** + * + * @param s + * @returns {string} + */ + soundex(s) { + const a = s.toLowerCase().split(''), + + codes = { + 'a': '', 'e': '', 'i': '', 'o': '', 'u': '', + 'b': 1, 'f': 1, 'p': 1, 'v': 1, + 'c': 2, 'g': 2, 'j': 2, 'k': 2, 'q': 2, 's': 2, 'x': 2, 'z': 2, + 'd': 3, 't': 3, + 'l': 4, + 'm': 5, 'n': 5, + 'r': 6 + }; + + const f = a.shift(); + let r = ''; + + r = f + + a + .map((v, i, a) => { + return codes[v]; + }) + .filter((v, i, a) => { + return ((i === 0) ? v !== codes[f] : v !== a[i - 1]); + }) + .join(''); + + return (`${r }000`).slice(0, 4).toUpperCase(); + }; + } module.exports = Scraper; diff --git a/ncas/at.js b/ncas/at.js index e75eb76..9ea1e88 100644 --- a/ncas/at.js +++ b/ncas/at.js @@ -89,9 +89,9 @@ class ATScrape extends Scraper { details['permissions'] = []; const permissionsDiv = $('div.modal-body'); $(permissionsDiv).find('h4').each((i, item) => { - const code = this._cleanUp($(item).text()); - const description = this._cleanUp($(item).next().text()); - details['permissions'].push({ 'code': code, 'description': description }); + const heading = this._cleanUp($(item).text()); + const body = $(item).next().html().split('
').map(x => this._cleanUp(x)).filter(x => x != ""); + details['permissions'].push({ heading, body }); }); } @@ -164,21 +164,19 @@ class ATScrape extends Scraper { const entities = $('div.company-details-wrap'); + const href = await this.page.url(); + entities.each(async (i, item) => { const noWhiteSpace = /\W/g; const details = this.extractEntityDetails($(item).html()); const id = this._makeFieldName(details.name); const entity = removeAccents.remove(id.trim()); - const filename = [this.modePrefix[this.mode], entity.replace(noWhiteSpace, '_')].join(''); + const filename = [this.modePrefix[this.mode], entity.replace(noWhiteSpace, '_'), '.json'].join(''); const filePath = `${this.path}/${filename}`.substring(0, 240); - jsonfile.writeFile(`${filePath}.json`, { details }); + jsonfile.writeFile(`${filePath}`, { details }); - this.getCurrentMode().links.push({ - 'id': id, - 'href': await this.page.url(), - 'filename': filename - }); + this.getCurrentMode().links.push({ id, href, filename }); }); logger.info(`${entities.length} ${this.modeNames[this.mode]} entities scraped.`); @@ -213,7 +211,6 @@ class ATScrape extends Scraper { await this.entityResultsPageProcessor(); else logger.error(`Page url not recognised: ${pageUrl.href}`); - } getCurrentMode() { @@ -234,12 +231,11 @@ class ATScrape extends Scraper { getNextUrl() { if (this.getCurrentMode().urlStep < this.getCurrentMode().urls.length - 1) this.getCurrentMode().urlStep++; - else { - if (this.mode < this.modeNames.length - 1) - this.mode++; - else - return null; - } + else + if (this.mode < this.modeNames.length - 1) + this.mode++; + else + return null; return this.getCurrentMode().urls[this.getCurrentMode().urlStep]; } diff --git a/ncas/be.js b/ncas/be.js index 243e369..06dcaf9 100644 --- a/ncas/be.js +++ b/ncas/be.js @@ -268,9 +268,9 @@ class BEScrape extends Scraper { const id = this.getIdByEntityName(entity.name); // create json file for each entity - const filename = [this.modePrefix[this.mode], id].join(''); + const filename = [this.modePrefix[this.mode], id, '.json'].join(''); const filePath = `${this.path}/${filename}`.substring(0, 240); - jsonfile.writeFile(`${filePath}.json`, { 'details': entity , metadataFileName}); + jsonfile.writeFile(filePath, { 'details': entity , metadataFileName}); // add entity details to "links" so that index file can be generated later this.getCurrentMode().links.push({ diff --git a/ncas/cy.js b/ncas/cy.js index 8bdb086..b699a18 100644 --- a/ncas/cy.js +++ b/ncas/cy.js @@ -15,6 +15,8 @@ class CYScrape extends Scraper { super(); this.setID('CY'); + this.addToBlockFilters(['recaptcha']); + this.on('done', () => { this._done(); }); diff --git a/ncas/cz.js b/ncas/cz.js index c6dee97..3a17cab 100644 --- a/ncas/cz.js +++ b/ncas/cz.js @@ -11,7 +11,7 @@ class CZScrape extends Scraper { constructor() { super(); - this.id = 'CZ'; + this.setID('CZ'); this.version = '0.0.1-3'; this.captchas = ['iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAj0lEQVRYR+2XQQ6AIAwE7f8fXeOBBIlll8bYSNZzhemUZMHc3Y/CzwSQMWBm06GtTDU1ghIAtGmkBNmgDZQBZDcejUQmoIEIYKb26Z/XANBMW+cjhABkQAZkQAZkYB8DLe0+i+PVeL3q+yhG8Q0vJBEA+5bZB6DvGN0TUde3tX75MGHnz9TRh5BZLFNTDnACDZUAsJw5oEAAAAAASUVORK5CYII=', @@ -218,23 +218,27 @@ class CZScrape extends Scraper { */ async entityCompleter(serviceObject) { let cbFlag = false; + try{ + if (serviceObject.current.authLink !== '' && !serviceObject.current.authProcess) { + await this._randomWait(this.page, 3, 5, 'Get Authorisations'); - if (serviceObject.current.authLink !== '' && !serviceObject.current.authProcess) { - await this._randomWait(this.page, 3, 5, 'Get Authorisations'); + await this._goto(serviceObject.current.authLink, { 'waitUntil':'networkidle0' }); - await this._goto(serviceObject.current.authLink, { 'waitUntil':'networkidle2' }); + return null; + } - return null; + if (typeof serviceObject.current.crossBorderLinks !== 'undefined' && !serviceObject.current.cbProcess && serviceObject.current.crossBorderLinks.length > 0) { + await this._randomWait(this.page, 3, 5, 'Get CBs'); + // logger.info(`Crossborder for ${serviceObject.current.crossBorderLinks[serviceObject.current.crossBorderStep].name}`); + + await this._goto(serviceObject.current.crossBorderLinks[serviceObject.current.crossBorderStep].href, { 'waitUntil':'networkidle0' }); + } + else + cbFlag = true; } - - if (typeof serviceObject.current.crossBorderLinks !== 'undefined' && !serviceObject.current.cbProcess && serviceObject.current.crossBorderLinks.length > 0) { - await this._randomWait(this.page, 3, 5, 'Get CBs'); - // logger.info(`Crossborder for ${serviceObject.current.crossBorderLinks[serviceObject.current.crossBorderStep].name}`); - - await this._goto(serviceObject.current.crossBorderLinks[serviceObject.current.crossBorderStep].href, { 'waitUntil':'networkidle2' }); + catch( err) { + logger.error(err); } - else - cbFlag = true; if( cbFlag === true) { const filename = serviceObject.links[serviceObject.step].fileName; @@ -254,7 +258,7 @@ class CZScrape extends Scraper { if (serviceObject.step < serviceObject.items) { serviceObject.current = {}; - await this._goto(serviceObject.links[serviceObject.step].href, { 'waitUntil':'networkidle2' }); + await this._goto(serviceObject.links[serviceObject.step].href, { 'waitUntil':'networkidle0' }); } else this.emit('serviceDone'); @@ -635,7 +639,7 @@ class CZScrape extends Scraper { await this._randomWait(this.page, 3, 5, 'First sub section'); - await this._goto(serviceObject.sectionLinks[serviceObject.indexStep], { 'waitUntil':'networkidle2' }); + await this._goto(serviceObject.sectionLinks[serviceObject.indexStep], { 'waitUntil':'networkidle0' }); } /** @@ -704,15 +708,16 @@ class CZScrape extends Scraper { if (serviceObject.indexStep >= serviceObject.sectionLinks.length) { this.inProgress = true; serviceObject.items = serviceObject.links.length; + await this._randomWait(this.page, 3, 5, 'First page'); logger.info('goto', serviceObject.links[serviceObject.step].href); - await this._goto(serviceObject.links[serviceObject.step].href, { 'waitUntil':'networkidle2' }); + await this._goto(serviceObject.links[serviceObject.step].href, { 'waitUntil':'networkidle0' }); } else { await this._randomWait(this.page, 3, 5, 'Next sub section'); - await this._goto(serviceObject.sectionLinks[serviceObject.indexStep], { 'waitUntil':'networkidle2', 'timeout': 0 }); + await this._goto(serviceObject.sectionLinks[serviceObject.indexStep], { 'waitUntil':'networkidle0', 'timeout': 5000 }); } } @@ -837,12 +842,13 @@ class CZScrape extends Scraper { */ async processNewPage() { // give the page a few seconds to settle + const errorPages = ['https://apl.cnb.cz/apljerrsdad/undefined', 'chrome-error://chromewebdata/']; await this._randomWait(this.page, 3, 5); const pageUrl = url.parse(await this.page.url()); - if (pageUrl.href === 'chrome-error://chromewebdata/') { - logger.warn('Directed to: chrome-error://chromewebdata/'); + if (errorPages.indexOf(pageUrl.href) !== -1) { + logger.warn(`Directed to: ${pageUrl.href}`); this.emit('recover'); return; @@ -880,7 +886,8 @@ class CZScrape extends Scraper { default: if (process.env.NODE_ENV) { await this._uploadError(); - throw new Error(`Unknown page: ${pageUrl}`); + // throw new Error(`Unknown page: ${pageUrl}`); + this.emit('recover'); } else { logger.warn('processNewPage Fell through'); @@ -1044,7 +1051,7 @@ class CZScrape extends Scraper { // await this.page.setViewport({ 'width': 1200, 'height': 800 }); - await this._goto(this.startPage, { 'waitUntil':'networkidle2' }); + await this._goto(this.startPage, { 'waitUntil':'networkidle0' }); await this._randomWait(this.page, 3, 5); } diff --git a/ncas/de.js b/ncas/de.js index 71ce77e..75cda76 100644 --- a/ncas/de.js +++ b/ncas/de.js @@ -5,7 +5,7 @@ const cheerio = require('cheerio'); const path = require('path'); const jsonfile = require('jsonfile'); const removeAccents = require('remove-accents-diacritics'); -const logger = require('log4js').getLogger('DE'); +const logger = require('log4js').getLogger('(DE)'); const url = require('url'); logger.level = process.env.LOGGER_LEVEL || 'warn'; @@ -123,94 +123,106 @@ class DEScrape extends Scraper { } async processCreditInstIndexPage() { - const noWhiteSpace = /\W/g; - logger.info('Building CI sub-index...'); + try{ + const noWhiteSpace = /\W/g; + logger.info('Building CI sub-index...'); - const wantedRowType = ['CRR-Kreditinstitut']; - const currentPage = await this.page.evaluate(() => document); - const body = await this.page.content(); - const $ = cheerio.load(body); + const wantedRowType = ['CRR-Kreditinstitut']; + const currentPage = await this.page.evaluate(() => document); + const body = await this.page.content(); + const $ = cheerio.load(body); - const search = currentPage.location.search; - const params = this._getParamsFromUrl(search); + const search = currentPage.location.search; + const params = this._getParamsFromUrl(search); - const currentPageID = params['d-4012550-p'] || ''; + const currentPageID = params['d-4012550-p'] || ''; - await this._makeScreenshotV2(this.page, `${this.path}/credit_instititute_menu_${currentPageID}`, null); + await this._makeScreenshotV2(this.page, `${this.path}/credit_instititute_menu_${currentPageID}`, null); - await this._randomWait(this.page, 7, 10); + await this._randomWait(this.page, 7, 10); - const rows = $('#institut tr'); + const rows = $('#institut tr'); - rows.each((i, elm) => { - const rowClass = cheerio(elm).attr('class'); + rows.each((i, elm) => { + const rowClass = cheerio(elm).attr('class'); - if (typeof(rowClass) !== 'undefined') { - const children = cheerio(elm).children(); + if (typeof(rowClass) !== 'undefined') { + const children = cheerio(elm).children(); - const rowType = children.eq(1).text(); + const rowType = children.eq(1).text(); - if (wantedRowType.indexOf(rowType) !== -1) { - const name = this._cleanUp(children.eq(0).text()); - const id = this._makeFieldName(name); - let href = cheerio(children.eq(0)).find('a').attr('href'); - const params = this._getParamsFromUrl(href); - href = href.concat('&locale=en_GB'); + if (wantedRowType.indexOf(rowType) !== -1) { + const name = this._cleanUp(children.eq(0).text()); + const id = this._makeFieldName(name); + let href = cheerio(children.eq(0)).find('a').attr('href'); + const params = this._getParamsFromUrl(href); + href = href.concat('&locale=en_GB'); - // this is the one we want. + // this is the one we want. - this.creditServices.links.push({ name, id, href, params }); + this.creditServices.links.push({ name, id, href, params }); + } } + }); + + const clicked = await this._findAndClick('.pagelinks a', 'Next'); + if (!clicked) { + // come to the end of the index.. + + this.creditServices.done = true; + this.creditServices.items = this.creditServices.links.length; + + this.emit('ciindexdone'); } - }); - - const clicked = await this._findAndClick('.pagelinks a', 'Next'); - if (!clicked) { - // come to the end of the index.. - - this.creditServices.done = true; - this.creditServices.items = this.creditServices.links.length; - - this.emit('ciindexdone'); + } + catch( err) { + logger.error(err); + this.emit('recover'); } } async processCreditInstPage() { - const noWhiteSpace = /\W/g; + try{ + const noWhiteSpace = /\W/g; - const id = this.creditServices.links[this.creditServices.step].id; - const name = this.creditServices.links[this.creditServices.step].name; - logger.info(`Process Credit Service entity ${this.creditServices.step} of ${this.creditServices.items} // ${name}`); + const id = this.creditServices.links[this.creditServices.step].id; + const name = this.creditServices.links[this.creditServices.step].name; + logger.info(`Process Credit Service entity ${this.creditServices.step} of ${this.creditServices.items} // ${name}`); - await this._randomWait(this.page, 3, 5); + await this._randomWait(this.page, 3, 5); - const body = await this.page.content(); + const body = await this.page.content(); - const details = await this.extractPaymentEntity(body); + const details = await this.extractPaymentEntity(body); - const entity = removeAccents.remove(details.description[0].trim()); + const entity = removeAccents.remove(details.description[0].trim()); - const filename = id.indexOf('?id=') === 0 ? this._makeFileName(entity) : this._makeFileName(id); + const filename = id.indexOf('?id=') === 0 ? this._makeFileName(entity) : this._makeFileName(id); - logger.debug('filename', filename); + logger.debug('filename', filename); - const filePath = `${this.path}/${filename}`.substring(0, 240); + const filePath = `${this.path}/${filename}`.substring(0, 240); - await this._makeScreenshotV2(this.page, `${filePath}_main`, null); + await this._makeScreenshotV2(this.page, `${filePath}_main`, null); - jsonfile.writeFileSync(`${filePath}.json`, details); + jsonfile.writeFileSync(`${filePath}.json`, details); - this.creditServices.links[this.creditServices.step].filename = `${filename}.json`; - this.creditServices.links[this.creditServices.step].filePath = `${filePath}`; - this.creditServices.step++; + this.creditServices.links[this.creditServices.step].filename = `${filename}.json`; + this.creditServices.links[this.creditServices.step].filePath = `${filePath}`; + this.creditServices.step++; - if (this.creditServices.step < this.creditServices.items) { - const newUrl = `https://portal.mvp.bafin.de/database/InstInfo/${this.creditServices.links[this.creditServices.step].href}`; + if (this.creditServices.step < this.creditServices.items) { + const newUrl = `https://portal.mvp.bafin.de/database/InstInfo/${this.creditServices.links[this.creditServices.step].href}`; - await this._goto(newUrl); + await this._goto(newUrl); + } + else + this.emit('creditinstdone'); + } + catch( err) { + logger.error(err); + this.emit('recover'); } - else - this.emit('creditinstdone'); } /** @@ -232,58 +244,64 @@ class DEScrape extends Scraper { * @returns {Promise<{description: T[] | jQuery, permissions: {original: Array, translated: Array}}>} */ async extractPaymentEntity(html) { - const permissions = { 'original':[], 'translated':[] }; + try{ + const permissions = { 'original':[], 'translated':[] }; - const newLine = /\n/g; - const $ = cheerio.load(html); + const newLine = /\n/g; + const $ = cheerio.load(html); - let description = $('#content > p').text().split(newLine).filter(line => line.length > 0); + let description = $('#content > p').text().split(newLine).filter(line => line.length > 0); - description = description.map((i) => { - return this._cleanUp(i.replace(/\t/g, '')).trim(); - }); + description = description.map((i) => { + return this._cleanUp(i.replace(/\t/g, '')).trim(); + }); - description = description.filter(item => item.length > 0); + description = description.filter(item => item.length > 0); - const rows = $('#erlaubnis > tbody tr'); + const rows = $('#erlaubnis > tbody tr'); - rows.each((index, item) => { - const cells = $(item).find('td'); + rows.each((index, item) => { + const cells = $(item).find('td'); - const service = $(cells.get(0)).text(); - const startAuth = $(cells.get(1)).text(); - const endAuth = $(cells.get(2)).text(); + const service = $(cells.get(0)).text(); + const startAuth = $(cells.get(1)).text(); + const endAuth = $(cells.get(2)).text(); - const reason = (cells.length === 4) ? $(cells.get(3)).text() : ''; + const reason = (cells.length === 4) ? $(cells.get(3)).text() : ''; - const phrasing = service.split(' (§'); - const translated = this._translate(phrasing[0]); + const phrasing = service.split(' (§'); + const translated = this._translate(phrasing[0]); - phrasing[0] = (translated !== '') ? translated : phrasing[0]; + phrasing[0] = (translated !== '') ? translated : phrasing[0]; - const newObjTrans = { - 'service': phrasing.join(' (§'), - startAuth, - endAuth - }; + const newObjTrans = { + 'service': phrasing.join(' (§'), + startAuth, + endAuth + }; - const newObj = { - service, - startAuth, - endAuth - }; + const newObj = { + service, + startAuth, + endAuth + }; - if (cells.length === 4) { - newObj.reason = reason; - newObjTrans.reason = reason; - } + if (cells.length === 4) { + newObj.reason = reason; + newObjTrans.reason = reason; + } - permissions.translated.push(newObjTrans); + permissions.translated.push(newObjTrans); - permissions.original.push(newObj); - }); + permissions.original.push(newObj); + }); - return { description, permissions }; + return { description, permissions }; + } + catch( err) { + logger.error(err); + this.emit('recover'); + } } /** @@ -291,58 +309,64 @@ class DEScrape extends Scraper { * @returns {Promise} */ async processEntity() { - const noWhiteSpace = /\W/g; - if (!this.subIndex.done) { - // We should not be here quite yet, so add this to subindex; - const currentPage = await this.page.evaluate(() => document); + try{ + const noWhiteSpace = /\W/g; + if (!this.subIndex.done) { + // We should not be here quite yet, so add this to subindex; + const currentPage = await this.page.evaluate(() => document); - const location = currentPage.location; - const id = location.search; - let href = location.href; - href = href.concat('&locale=en_GB'); + const location = currentPage.location; + const id = location.search; + let href = location.href; + href = href.concat('&locale=en_GB'); - this.paymentServices.links.push({ id, href }); + this.paymentServices.links.push({ id, href }); - this.index.step++; + this.index.step++; - if (this.index.step < this.index.items) - this.emit('nextsubindex'); + if (this.index.step < this.index.items) + this.emit('nextsubindex'); + else { + logger.info('Sub indexing done...'); + this.subIndex.done = true; + this.paymentServices.items = this.paymentServices.links.length; + this.emit('subindexdone'); + } + } else { - logger.info('Sub indexing done...'); - this.subIndex.done = true; - this.paymentServices.items = this.paymentServices.links.length; - this.emit('subindexdone'); + const id = this.paymentServices.links[this.paymentServices.step].id; + // logger.info('Process entity:', id); + logger.info(`Process entity ${this.paymentServices.step} of ${this.paymentServices.items} // ${id}`); + await this._randomWait(this.page, 3, 5); + + const body = await this.page.evaluate(() => document.documentElement.outerHTML); + + const details = await this.extractPaymentEntity(body); + + const entity = removeAccents.remove(details.description[0].trim()); + + // const filename = id.indexOf('?id=') === 0 ? `ps_${entity.replace(noWhiteSpace, '_')}` : `ps_${id.replace(noWhiteSpace, '_')}`; + + const filename = id.indexOf('?id=') === 0 ? this._makeFileName(entity) : this._makeFileName(id); + + logger.debug('filename', filename); + + await this._makeScreenshotV2(this.page, `${this.path}/${filename}_main`, null); + + jsonfile.writeFileSync(`${this.path}/${filename}.json`, details); + this.paymentServices.links[this.paymentServices.step].filename = `${filename}.json`; + + this.paymentServices.step++; + + if (this.paymentServices.step < this.paymentServices.items) + await this._goto(this.paymentServices.links[this.paymentServices.step].href); + else + this.emit('processdone'); } } - else { - const id = this.paymentServices.links[this.paymentServices.step].id; - // logger.info('Process entity:', id); - logger.info(`Process entity ${this.paymentServices.step} of ${this.paymentServices.items} // ${id}`); - await this._randomWait(this.page, 3, 5); - - const body = await this.page.evaluate(() => document.documentElement.outerHTML); - - const details = await this.extractPaymentEntity(body); - - const entity = removeAccents.remove(details.description[0].trim()); - - // const filename = id.indexOf('?id=') === 0 ? `ps_${entity.replace(noWhiteSpace, '_')}` : `ps_${id.replace(noWhiteSpace, '_')}`; - - const filename = id.indexOf('?id=') === 0 ? this._makeFileName(entity) : this._makeFileName(id); - - logger.debug('filename', filename); - - await this._makeScreenshotV2(this.page, `${this.path}/${filename}_main`, null); - - jsonfile.writeFileSync(`${this.path}/${filename}.json`, details); - this.paymentServices.links[this.paymentServices.step].filename = `${filename}.json`; - - this.paymentServices.step++; - - if (this.paymentServices.step < this.paymentServices.items) - await this._goto(this.paymentServices.links[this.paymentServices.step].href); - else - this.emit('processdone'); + catch( err) { + logger.error(err); + this.emit('reover'); } } @@ -453,12 +477,12 @@ class DEScrape extends Scraper { * @returns {Promise} */ async attachEvents() { - this.on('startcredit', async function() { + this.on('startcredit', async () => { logger.info('Starting Credit Institutes'); await this._goto(this.credit); }); - this.on('processdone', async function() { + this.on('processdone', async () => { logger.warn('Payment Entities done', this.paymentServices.items); jsonfile.writeFileSync(`${this.path}/paymentServices.json`, { 'links': this.paymentServices.links }); @@ -469,19 +493,19 @@ class DEScrape extends Scraper { await this._goto(this.emoneyUrl); }); - this.on('subindexdone', async function() { + this.on('subindexdone', async () => { logger.info('Sub Index done', this.paymentServices.items); logger.info(this.paymentServices.links[this.paymentServices.step].href); await this._goto(this.paymentServices.links[this.paymentServices.step].href); }); - this.on('indexdone', async function() { + this.on('indexdone', async () => { logger.info('Index done', this.index.items); logger.info(this.index.links[this.index.step].href); await this._goto(this.index.links[this.index.step].href); }); - this.on('ciindexdone', async function() { + this.on('ciindexdone', async () => { logger.info('CI Index done', this.creditServices.items); logger.info(this.creditServices.links[this.creditServices.step].href); @@ -489,7 +513,7 @@ class DEScrape extends Scraper { await this._goto(newUrl); }); - this.on('creditinstdone', async function() { + this.on('creditinstdone', async () => { logger.debug('Credit Institutes done', this.paymentServices.items); jsonfile.writeFileSync(`${this.path}/creditServices.json`, { 'links':this.creditServices.links }); @@ -499,7 +523,7 @@ class DEScrape extends Scraper { this.emit('done'); }); - this.on('nextsubindex', async function() { + this.on('nextsubindex', async () => { logger.debug(this.index.links[this.index.step].href); await this._goto(this.index.links[this.index.step].href); }); diff --git a/ncas/dkV2.js b/ncas/dkV2.js index f148427..d575513 100644 --- a/ncas/dkV2.js +++ b/ncas/dkV2.js @@ -1,6 +1,6 @@ const Scraper = require('../helpers/scraper'); const path = require('path'); -const logger = require('log4js').getLogger('DK'); +const logger = require('log4js').getLogger('(DK)'); const url = require('url'); logger.level = process.env.LOGGER_LEVEL || 'warn'; @@ -9,7 +9,7 @@ class DKScrape extends Scraper { constructor(checkForLock = true) { super(); - this.id = 'DK'; + this.setID('DK'); this.on('done', () => { this._done(); @@ -55,14 +55,20 @@ class DKScrape extends Scraper { * @returns {Promise} */ async renameFile() { - const filename = this.filenames[this.step]; + if (!this.errored) { + const filename = this.filenames[this.step]; - const sourceFile = 'Finanstilsynets virksomhedsregister - SQL.xlsx'; + const sourceFile = 'Finanstilsynets virksomhedsregister - SQL.xlsx'; - const origFile = `${this.path}/${sourceFile}`; - const newFile = `${this.path}/${filename}.xlsx`; + const origFile = `${this.path}/${sourceFile}`; + const newFile = `${this.path}/${filename}.xlsx`; - await this._renameFile(origFile, newFile); + await this._renameFile(origFile, newFile); + + await this._randomWait(this.page, 5, 7, 'after renameFile'); + } + else + logger.warn('Skipping renameFile'); } /** @@ -70,17 +76,26 @@ class DKScrape extends Scraper { * @returns {Promise} */ async clickReturn() { - logger.debug('clickReturn'); - await this.iframe.waitForSelector('#lsAnalysisPage > div > div:nth-child(2)', { 'visible':true, 'timeout':75000 }).then(async (elm) => { - console.log('found'); - await elm.click({ 'delay':90 }); - }).catch((e) => { - logger.error('iframe missing stuff', e); - // pageLoaded = false; - }); + if (!this.errored) { + logger.debug('clickReturn'); + await this.iframe.waitForSelector('#lsAnalysisPage > div > div:nth-child(2)', { 'visible':true, 'timeout':75000 }).then(async (elm) => { + console.log('found'); + await elm.click({ 'delay':90 }); + await this._randomWait(this.page, 5, 7, 'after clickReturn click'); + this.step++; + }).catch((e) => { + logger.error('iframe missing stuff (clickReturn)', e); + // pageLoaded = false; + this.emit('recover'); + }); + } + else + logger.warn('Skipping clickReturn'); - await this._randomWait(this.page, 2, 3, 'after clickReturn click'); + /* + await this._randomWait(this.page, 5, 7, 'after clickReturn click'); this.step++; +*/ } /** @@ -88,22 +103,26 @@ class DKScrape extends Scraper { * @returns {Promise} */ async scrollContainer() { - await this.page.evaluate(() => { - console.log('window.innerWidth', window.innerWidth); - window.scrollBy(window.innerWidth, window.innerHeight); - }); + if (!this.errored) { + await this.page.evaluate(() => { + console.log('window.innerWidth', window.innerWidth); + window.scrollBy(window.innerWidth, window.innerHeight); + }); - await this._randomWait(this.page, 2, 2, 'scroll x?'); + await this._randomWait(this.page, 2, 2, 'scroll x?'); - this.page.mouse.move(1061, 437); - await this._randomWait(this.page, 2, 3, 'bottom right scroll arrow'); + this.page.mouse.move(1061, 437); + await this._randomWait(this.page, 2, 3, 'bottom right scroll arrow'); - for(let count = 0; count < 15; count++) { - this.page.mouse.click(1061, 437, { 'delay':500 }); - await this._randomWait(this.page, 1, 2, 'scrolling'); + for(let count = 0; count < 15; count++) { + this.page.mouse.click(1061, 437, { 'delay':821 }); + await this._randomWait(this.page, 4, 7, 'scrolling'); + } + + await this._randomWait(this.page, 5, 7, 'after scroll'); } - - await this._randomWait(this.page, 4, 5, 'after scroll'); + else + logger.warn('Skipping scrollContainer'); } /** @@ -111,38 +130,43 @@ class DKScrape extends Scraper { * @returns {Promise} */ async clickExport() { - logger.debug('clickExport'); + if (!this.errored) { + logger.debug('clickExport'); - await this.movePageToTop(); + await this.movePageToTop(); - await this._randomWait(this.page, 2, 2, 'Move to top'); + await this._randomWait(this.page, 2, 2, 'Move to top'); - const filename = this.filenames[this.step]; + const filename = this.filenames[this.step]; - const filePath = `${this.path}/${filename}`.substring(0, 240); + const filePath = `${this.path}/${filename}`.substring(0, 240); - await this._makeScreenshotV2(this.page, `${filePath}_main`, null); + await this._makeScreenshotV2(this.page, `${filePath}_main`, null); - await this._randomWait(this.page, 4, 4, 'Screenshot'); + await this._randomWait(this.page, 4, 4, 'Screenshot'); - this.page.mouse.move(175, 440); - await this._randomWait(this.page, 2, 3, 'Move 175, 440'); + this.page.mouse.move(175, 440); + await this._randomWait(this.page, 2, 3, 'Move 175, 440'); - this.page.mouse.click(175, 440, { 'button':'right', 'delay':90 }); + this.page.mouse.click(175, 440, { 'button':'right', 'delay':90 }); - await this._randomWait(this.page, 2, 3, 'Click 175, 440'); + await this._randomWait(this.page, 2, 3, 'Click 175, 440'); - await this.page._client.send('Page.setDownloadBehavior', { 'behavior': 'allow', 'downloadPath': this.path }); + await this.page._client.send('Page.setDownloadBehavior', { 'behavior': 'allow', 'downloadPath': this.path }); - await this.iframe.waitForSelector('div.lsDialogContent > div:nth-child(2)', { 'visible':true, 'timeout':75000 }).then(async (elm) => { - console.log('found'); - await elm.click({ 'delay':90 }); - }).catch((e) => { - logger.error('iframe missing stuff', e); - // pageLoaded = false; - }); - - await this._randomWait(this.page, 2, 3, 'after clickExport click'); + await this.iframe.waitForSelector('div.lsDialogContent > div:nth-child(2)', { 'visible':true, 'timeout':75000 }).then(async (elm) => { + console.log('found'); + await elm.click({ 'delay':90 }); + await this._randomWait(this.page, 5, 7, 'after clickExport click'); + }).catch((e) => { + logger.error('iframe missing stuff (clickExport)', e); + this.errored = true; + this.emit('recover'); + // pageLoaded = false; + }); + } + else + logger.warn('Skipping clickExport'); } /** @@ -159,12 +183,13 @@ class DKScrape extends Scraper { await this.iframe.waitForSelector('#lsAnalysisPage > div > div:nth-child(11)', { 'visible':true, 'timeout':75000 }).then(async (elm) => { console.log('found'); await elm.click({ 'delay':90 }); + await this._randomWait(this.page, 5, 7, 'after clickSearch click'); }).catch((e) => { logger.error('iframe missing stuff', e); // pageLoaded = false; + this.emit('recover'); + this.errored = true; }); - - await this._randomWait(this.page, 2, 3, 'after clickSearch click'); } /** @@ -191,7 +216,7 @@ class DKScrape extends Scraper { this.page.mouse.click(400, 434); - await this._randomWait(this.page, 2, 3, 'Click 400, 434'); + await this._randomWait(this.page, 5, 7, 'Click 400, 434'); } /** @@ -219,7 +244,7 @@ class DKScrape extends Scraper { this.page.mouse.click(400, 585); - await this._randomWait(this.page, 2, 3, 'Click 400, 585'); + await this._randomWait(this.page, 5, 7, 'Click 400, 585'); } /** @@ -246,7 +271,7 @@ class DKScrape extends Scraper { this.page.mouse.click(400, 473); - await this._randomWait(this.page, 2, 3, 'Click 400, 473'); + await this._randomWait(this.page, 5, 7, 'Click 400, 473'); } /** @@ -273,7 +298,7 @@ class DKScrape extends Scraper { this.page.mouse.click(400, 631); - await this._randomWait(this.page, 2, 3, 'Click 400, 631'); + await this._randomWait(this.page, 5, 7, 'Click 400, 631'); } /** @@ -300,7 +325,7 @@ class DKScrape extends Scraper { this.page.mouse.click(400, 473); - await this._randomWait(this.page, 2, 3, 'Click 400, 473'); + await this._randomWait(this.page, 5, 7, 'Click 400, 473'); } /** @@ -308,7 +333,7 @@ class DKScrape extends Scraper { * @returns {Promise} */ async motions() { - do + do switch(this.step) { case 0: @@ -392,8 +417,22 @@ class DKScrape extends Scraper { break; } - - while(!this.complete ); + + while(!this.complete && !this.errored); + } + + /** + * + * @param restartURL + * @returns {Promise} + * @private + */ + async __recover() { + // need to reset the error status then restart + logger.info('OVERLOAD __recover'); + this.errored = false; + + super.__recover(this.startPage); } /** @@ -410,7 +449,7 @@ class DKScrape extends Scraper { await this.motions(); }).catch((e) => { - logger.error('processEntityDetails', e); + logger.error('waitForIframe', e); // pageLoaded = false; }); } @@ -448,6 +487,8 @@ class DKScrape extends Scraper { this.step = 0; this.complete = false; + this.errored = false; + this.filenames = ['paymentServices01', 'paymentServices02', 'eMoney01', 'eMoney02', 'creditServices01', 'creditServices02']; this.startPage = 'https://virksomhedsregister.finanstilsynet.dk/listeudtr%C3%A6k-en.html'; @@ -458,7 +499,7 @@ class DKScrape extends Scraper { logger.error(err); }); - await this._initBrowser(); + await this._initBrowser(false); await this._createBrowserPage(); await this._makeResponsive(); diff --git a/ncas/ee.js b/ncas/ee.js index faf22e3..6701625 100644 --- a/ncas/ee.js +++ b/ncas/ee.js @@ -2,7 +2,7 @@ const Scraper = require('../helpers/scraper'); const cheerio = require('cheerio'); const path = require('path'); const jsonfile = require('jsonfile'); -const logger = require('log4js').getLogger('EE'); +const logger = require('log4js').getLogger('(EE)'); const url = require('url'); const removeAccents = require('remove-accents-diacritics'); @@ -12,7 +12,7 @@ class EEScrape extends Scraper { constructor() { super(); - this.id = 'EE'; + this.setID('EE'); this.on('done', () => { this._done(); @@ -574,7 +574,7 @@ class EEScrape extends Scraper { if (this.paymentServices.indexStep >= this.paymentServices.urls.length) { this.workMode = 1; - logger.debug(JSON.stringify(this.paymentServices)); + // logger.debug(JSON.stringify(this.paymentServices)); newUrl = this.paymentServices.links[this.paymentServices.step].href; } else diff --git a/ncas/es.js b/ncas/es.js index f9922ec..a666323 100644 --- a/ncas/es.js +++ b/ncas/es.js @@ -1,7 +1,7 @@ const Scraper = require('../helpers/scraper'); const cheerio = require('cheerio'); const path = require('path'); -const logger = require('log4js').getLogger('ES'); +const logger = require('log4js').getLogger('(ES)'); const url = require('url'); const querystring = require('querystring'); const removeAccents = require('remove-accents-diacritics'); @@ -13,7 +13,7 @@ class ESScrape extends Scraper { constructor() { super(); - this.id = 'ES'; + this.setID('ES'); this.on('done', () => { this._done(); @@ -279,7 +279,7 @@ class ESScrape extends Scraper { const { name, id } = serviceObject.links[serviceObject.step]; logger.info(`Process ${this.modeTitles[this.mode]} entity ${serviceObject.step + 1} of ${serviceObject.items} // ${name}`); - await this.page.waitForSelector('td.tdContenido', { 'visible':true, 'timeout':7500 }); // Wait for buttons at bottom of table to be visible + await this.page.waitForSelector('td.tdContenido', { 'visible':true, 'timeout':90000 }); // Wait for buttons at bottom of table to be visible await this._randomWait(this.page, 3, 5); @@ -361,6 +361,7 @@ class ESScrape extends Scraper { await this.processRedirector(); else { await this._uploadError(); + this.emit('recover'); throw new Error(`Unknown page: ${pageUrl}`); } } @@ -370,7 +371,7 @@ class ESScrape extends Scraper { * @returns {Promise} */ async attachEvents() { - this.on('serviceDone', async function() { + this.on('serviceDone', async () => { switch (this.mode) { case 0: @@ -388,6 +389,8 @@ class ESScrape extends Scraper { } }); + // + this.on('psindexdone', async () => { this.paymentServices.items = this.paymentServices.links.length; logger.info(`${this.paymentServices.items} paymentServices items indexed`); @@ -397,6 +400,8 @@ class ESScrape extends Scraper { await this._goto(newUrl); }); + // + this.on('emindexdone', async () => { this.emoneyServices.items = this.emoneyServices.links.length; logger.info(`${this.emoneyServices.items} emoneyServices items indexed`); @@ -406,6 +411,8 @@ class ESScrape extends Scraper { await this._goto(newUrl); }); + // + this.on('ciindexdone', async () => { this.creditServices.items = this.creditServices.links.length; logger.info(`${this.creditServices.items} creditServices items indexed`); @@ -415,6 +422,8 @@ class ESScrape extends Scraper { await this._goto(newUrl); }); + // + this.on('indexdone', async () => { switch (this.mode) { diff --git a/ncas/it.js b/ncas/it.js index d4470f5..7db105e 100644 --- a/ncas/it.js +++ b/ncas/it.js @@ -2,7 +2,7 @@ const Scraper = require('../helpers/scraper'); const cheerio = require('cheerio'); const path = require('path'); const jsonfile = require('jsonfile'); -const logger = require('log4js').getLogger('IT'); +const logger = require('log4js').getLogger('(IT)'); const url = require('url'); logger.level = process.env.LOGGER_LEVEL || 'warn'; @@ -56,16 +56,21 @@ class ITscrape extends Scraper { * @returns {Promise} */ async forceEnglish() { - await this._randomWait(this.page, 2, 2, 'Force English'); + const localStorage = await this._getLocalStorage(); - await this.page.waitForSelector('#bs-example-navbar-collapse-1 > ul > li.dropdown > a', { 'visible':true, 'timeout':7500 }).then(async (elm) => { - await elm.click({ 'delay':Scraper.notARobot() }); - await this._randomWait(this.page, 2, 2); - }).catch(() => { - logger.debug('No Language button'); - }); + const userLang = localStorage['userLang'].trim(); - await this._findAndClick('#bs-example-navbar-collapse-1 > ul > li.dropdown.open > ul > li:nth-child(2) > a'); + if (userLang !== 'en') { + await this._randomWait(this.page, 2, 2, 'Force English'); + await this.page.waitForSelector('#bs-example-navbar-collapse-1 > ul > li.dropdown > a', { 'visible':true, 'timeout':7500 }).then(async (elm) => { + await elm.click({ 'delay':Scraper.notARobot() }); + await this._randomWait(this.page, 2, 2); + }).catch(() => { + logger.debug('No Language button'); + }); + + await this._findAndClick('#bs-example-navbar-collapse-1 > ul > li.dropdown.open > ul > li:nth-child(2) > a'); + } } /** @@ -74,6 +79,8 @@ class ITscrape extends Scraper { */ async handleFrontPage() { let pageReturned = false; + this.processing = false; + await this._randomWait(this.page, 3, 5, 'handleFrontPage'); await this.page.waitFor('ul.linkgroup a', { 'visible':true }).then(async (elm) => { @@ -86,16 +93,16 @@ class ITscrape extends Scraper { await this.page.waitFor('#my-container > div.container > div', { 'visible':true, 'timeout':7500 }).then(() => { pageReturned = true; }).catch(async () => { - logger.info('We didnt transition back correctly, forcing another click..\n'); + logger.info('(handleFrontPage) We didnt transition back correctly, forcing another click..\n'); }); if (!pageReturned) { await this.page.hover('ul.linkgroup a').catch((err) => { - logger.debug(err.name); + logger.debug(err.name, 'hover ul.linkgroup a'); }); await this.page.focus('ul.linkgroup a').catch((err) => { - logger.debug(err.name); + logger.debug(err.name, 'focus ul.linkgroup a'); }); await this.page.waitFor('ul.linkgroup a', { 'visible':true }).then(async (elm) => { @@ -106,7 +113,7 @@ class ITscrape extends Scraper { } } - while(!pageReturned); + while(!pageReturned && !this.processing); // Supervisory registers and lists } @@ -126,6 +133,11 @@ class ITscrape extends Scraper { logger.warn('Ajax loading shroud not removed after 90 seconds'); }); + await this.page.waitForSelector('div.loading', { 'visible':true, 'timeout':10000 }).catch((e) => { + logger.warn('Ajax loading shroud still there...'); + }); + // body > app-root > div:nth-child(2) > div + await this.page.waitForSelector('ul.nav.navbar-nav.navbar-center li a', { 'visible':false, 'timeout':90000 }).then(async (elm) => { await elm.click({ 'delay':Scraper.notARobot() }); await this._randomWait(this.page, 5, 8, 'await transition'); @@ -291,17 +303,18 @@ class ITscrape extends Scraper { await this.page.waitForSelector('div.loading', { 'visible':false, 'timeout':25000 }); let btnSuccess = false; - + let breakCount = 0; do { - await this.page.waitForSelector('button.btn.btn-success', { 'visible':true, 'timeout':2500 }).then(async (elm) => { + await this.page.waitForSelector('button.btn.btn-success', { 'visible':true, 'timeout':45000 }).then(async (elm) => { await elm.click({ 'delay':Scraper.notARobot() }); }).catch(() => { btnSuccess = true; }); await this._randomWait(this.page, 1, 1, 'preparePSSearch btnSuccess'); + breakCount++; } - while(!btnSuccess); + while(!btnSuccess && breakCount < 5); this.page.waitFor('app-int-albi-grid-result').then(async () => { // @@ -328,66 +341,30 @@ class ITscrape extends Scraper { /** * - * @returns {Promise<{registry, authority, registers}>} + * @param filePath + * @returns {Promise} */ - async processPSDetail() { - let registry = {}, registers = {}, authority = {}; + async processPSDetailV2(filePath) { + await this._randomWait(this.page, 1, 3, 'processPSDetailV2: Screenshots'); - await this._randomWait(this.page, 3, 3, 'processPSDetail: AJAX'); - - // await this._makeScreenshotV2(this.page, `${filePath}_main`, null); - - await this.page.waitFor('#sub-navbar > app-int-albi > app-int-albi-details > div > div.card.card-title > span > span', { 'visible': true }).catch((err) => { - logger.warn('AJAX data has failed to load'); - logger.debug(err); - - return { registry, registers, authority }; - }); - - await this.page.waitFor('app-int-albi-details').then(async () => { - await this.forceScrollToTop(); - - const body = await this.page.content(); - - registry = await this.extractPSRegistry(body); - - await this._randomWait(this.page, 2, 2, 'processPSDetail app-int-albi-details'); - }).catch(async (err) => { - if (process.env.NODE_ENV) { - await this._uploadError(); - throw new Error('processPSDetail\n', err); - } - else - logger.error('processPSDetail\n', err); - }); - - await this._randomWait(this.page, 1, 1, 'processPSDetail after app-int-albi-details'); - - // + // wait for the data await this.forceScrollToTop(); + await this._makeScreenshotV2(this.page, `${filePath}_detail`, null); // wait for Registers Tab - await this.page.waitFor('#sub-navbar > app-int-albi > app-int-albi-details > div > div:nth-child(2) > ul > li:nth-child(2) > a', { 'visible': true, 'timeout':10000 }).then(async (elm) => { - logger.debug('** Showing Registers Tab'); + await this.page.waitFor('#sub-navbar > app-int-albi > app-int-albi-details > div > div:nth-child(2) > ul > li:nth-child(2) > a', + { 'visible': true, 'timeout':10000 }).then(async (elm) => { await elm.click({ 'delay':90 }); + await this._randomWait(this.page, 1, 2, 'processPSDetailV2 wait for Tab trans'); + await this.page.waitFor('app-details-albi', { 'visible': true, 'timeout':10000 }).then(async () => { - const body = await this.page.content(); - - registers = await this.extractPSRegisters(body); - - await this._randomWait(this.page, 2, 2, 'processPSDetail app-details-albi'); - }).catch(async (err) => { - if (process.env.NODE_ENV) - // await this._uploadError(); - throw new Error('No tab transition\n', err); - - else - logger.error('No tab transition'); + await this._makeScreenshotV2(this.page, `${filePath}_reg`, null); + await this._randomWait(this.page, 3, 5, 'processPSDetailV2 ss'); + }).catch(async () => { + logger.info('No tab transition'); }); - - await this._randomWait(this.page, 1, 1, 'processPSDetail after app-details-albi'); }).catch((err) => { logger.warn('No "registers" Block...'); logger.debug(err); @@ -395,36 +372,29 @@ class ITscrape extends Scraper { // wait for Activity Tab await this.forceScrollToTop(); - await this.page.waitFor('#sub-navbar > app-int-albi > app-int-albi-details > div > div:nth-child(2) > ul > li:nth-child(3) > a', { 'visible': true, 'timeout':10000 }).then(async (elm) => { - logger.debug('** Showing Activity Tab'); + await this.page.waitFor('#sub-navbar > app-int-albi > app-int-albi-details > div > div:nth-child(2) > ul > li:nth-child(3) > a', + { 'visible': true, 'timeout':10000 }).then(async (elm) => { await elm.click({ 'delay':90 }); let pageReturned = false; - + await this._randomWait(this.page, 1, 2, 'processPSDetailV2 wait for Tab B trans'); do await this.page.waitFor('app-details-att-autorizzate', { 'visible': true, 'timeout':10000 }).then(async () => { pageReturned = true; - const body = await this.page.content(); - authority = await this.extractPSAuthority(body); - - await this._randomWait(this.page, 2, 2, 'processPSDetail app-details-att-autorizzate'); - }).catch(async (err) => { + // screen shot + await this._makeScreenshotV2(this.page, `${filePath}_activity`, null); + await this._randomWait(this.page, 3, 5, 'processPSDetailV2 ss'); + }).catch(async () => { await this.forceScrollToTop(); await this._findAndClick('#sub-navbar > app-int-albi > app-int-albi-details > div > div:nth-child(2) > ul > li:nth-child(3) > a'); - if (process.env.NODE_ENV) - throw new Error('No tab transition\n', err); - - else - logger.warn('No tab transition'); + logger.info('No tab transition'); }); while(!pageReturned); }).catch((err) => { logger.warn('No "Activity" Block...'); logger.debug(err); }); - - return { registry, registers, authority }; } /** @@ -443,8 +413,9 @@ class ITscrape extends Scraper { await this.page.waitFor('app-int-albi-grid-result').then(() => { pageReturned = true; }).catch(async (err) => { - logger.warn('We didnt transition back correctly, forcing another click..\n', err); + logger.warn('(returnToPSList) We didnt transition back correctly, forcing another click..\n', err); + await this._randomWait(this.page, 1, 1, 'one sec'); await this.forceScrollToTop(); await this._findAndClick('#sub-navbar > giava-breadcrumb > ol > li:nth-child(3) > a'); @@ -482,7 +453,6 @@ class ITscrape extends Scraper { divs.each((index, item) => { const itemText = this._cleanUp($(item).text()); const itemName = $(item).attr('col-id'); - // logger.info(`>> ${index}`, itemName, itemText); entries[itemName] = itemText; }); @@ -521,6 +491,8 @@ class ITscrape extends Scraper { let maxPages = 0; let rowsInPass; + this.processing = true; + await this.psSetListCount(_defaultMaxPerPage); const maxRows = await this.psGetMaxRows(); let remainingRows = maxRows; @@ -594,32 +566,38 @@ class ITscrape extends Scraper { serviceObject.links.push({ uid, 'fileName':`${fileName}.json`, 'name':clickName }); // Go into the detail - await clickable.click(); + await clickable.click().catch((e) => { + logger.debug('something happend with the click', e); + throw new Error(e); + }); await this._randomWait(this.page, 3, 4, 'processAGTableV3 before next'); remainingRows--; - await this.page.waitFor('app-int-albi-details').then( - - await this.doAlbiDetails(filePath, newEntry) - - ).catch(async (err) => { + await this.page.waitFor('app-int-albi-details').catch(async (err) => { logger.error('No detail transition', err); this.emit('recover'); if (process.env.NODE_ENV) await this._uploadError(); }); + + await this.doAlbiDetailsV2(filePath, newEntry).catch(async (err) => { + logger.error('doAlbiDetailsV2 had an issue', err); + this.emit('recover'); + }); } } } - if (maxPages > 0) { + if (maxPages > 0 && !this.warning) { logger.info('Clicking to the next page...'); const nextButton = await this.page.$$('#sub-navbar > app-int-albi > app-int-albi-grid-result > grid-pagination > div > div > div:nth-child(5) > button'); - const buttonDisabled = await this.page.evaluate(el => el.disabled, nextButton[0]); + const buttonDisabled = await this.page.evaluate(el => el.disabled, nextButton[0]).catch((e) => { + logger.warn('Could not mark the button as disabled'); + }); if (!buttonDisabled) { this._findAndClick('#sub-navbar > app-int-albi > app-int-albi-grid-result > grid-pagination > div > div > div:nth-child(5) > button'); await this._randomWait(this.page, 5, 5, 'processAGTableV3 next page click'); @@ -632,30 +610,64 @@ class ITscrape extends Scraper { this.emit('doneProcessingGrid'); } - async doAlbiDetails(filePath, newEntry) { + /** + * + * @param filePath + * @param newEntry + * @returns {Promise} + */ + async doAlbiDetailsV2(filePath, newEntry) { + const empty = { }; + + logger.debug('++ doAlbiDetailsV2'); + let data; try{ // process the page - const data = await this.processPSDetail(); - data.details = newEntry; - logger.info(`Saving ${filePath}.json`); - await jsonfile.writeFile(`${filePath}.json`, data); + let breaker = 0; - await this._randomWait(this.page, 5, 7, 'doAlbiDetails'); + do{ + await this._randomWait(this.page, 1, 3, 'Retrieve from localStorage'); + const localStorage = await this._getLocalStorage(); - // Retun back to list + logger.debug('>> typeof(localStorage[\'intermediaryDetails\'])', typeof(localStorage['intermediaryDetails'])); - await this.returnToPSList(); + if (typeof(localStorage['intermediaryDetails']) !== 'undefined' ) + data = JSON.parse(localStorage['intermediaryDetails']) ; + else + await this._randomWait(this.page, 2, 2, 'wait for the localStorage load'); + + logger.debug('>> typeof(data)', typeof(data)); + breaker++; + } + while((typeof(data) === 'undefined' || data === null) && breaker < 10); + + if (typeof(data) === 'undefined' || JSON.stringify(empty) === JSON.stringify(data)) { + logger.warn('localStorage.intermediaryDetails did not load'); + + throw new Error('localStorage.intermediaryDetails did not load'); + } + else { + await this.processPSDetailV2(filePath); + + logger.info(`Saving ${filePath}.json`); + await jsonfile.writeFile(`${filePath}.json`, data); + + await this._randomWait(this.page, 5, 7, 'doAlbiDetailsV2'); + + // Retun back to list + + await this.returnToPSList(); + + await this._randomWait(this.page, 2, 2, 'doAlbiDetailsV2 after returnToPSList'); + } - await this._randomWait(this.page, 2, 2, 'doAlbiDetails after returnToPSList'); // wArray.push([uid, clickName]); } catch (err) { - logger.error('doAlbiDetails\n', err); + logger.error('doAlbiDetailsV2\n', err); + logger.debug('Converted data:', data); this.emit('recover'); - - if (process.env.NODE_ENV) - await this._uploadError(); } } @@ -719,9 +731,11 @@ class ITscrape extends Scraper { }, 2500)); this.on('recover', this._debounce(async () => { + this.warning = true; clearTimeout(this.backOffTimer); logger.warn('Backing off for 5 minutes..'); + this.processing = false; const timeout = (60 * 1000) * 5; this.backOffTimer = setTimeout(() => { @@ -731,6 +745,7 @@ class ITscrape extends Scraper { }, 30000)); this.on('restart', this._debounce(async() => { + this.warning = false; clearTimeout(this.backOffTimer); logger.warn('Restarting::'); @@ -804,7 +819,9 @@ class ITscrape extends Scraper { async start() { super._start(); try { - this.mode = 0; + this.mode = 2; + this.processing = false; + this.warning = false; this.modeTitles = ['Payment Service', 'EMoney', 'Credit Services']; @@ -846,7 +863,7 @@ class ITscrape extends Scraper { 'workingIndex': 0 }; - this.startPage = this.paymentServices.urls[0]; + this.startPage = this.creditServices.urls[0]; this.emoneyUrl = ''; this.credit = ''; @@ -858,7 +875,7 @@ class ITscrape extends Scraper { logger.warn(err); }); - await this._initBrowser(true); + await this._initBrowser(); await this._createBrowserPage(); this.page.on('domcontentloaded', this._throttle(async () => { diff --git a/ncas/lu.js b/ncas/lu.js index 78e8204..515335c 100644 --- a/ncas/lu.js +++ b/ncas/lu.js @@ -159,6 +159,7 @@ class LUScrape extends Scraper { } catch( err) { logger.error(err); + this.emit('recover'); } } diff --git a/ncas/lv.js b/ncas/lv.js index 5f7aa07..d488b2b 100644 --- a/ncas/lv.js +++ b/ncas/lv.js @@ -2,7 +2,7 @@ const Scraper = require('../helpers/scraper'); const cheerio = require('cheerio'); const path = require('path'); const jsonfile = require('jsonfile'); -const logger = require('log4js').getLogger('LV'); +const logger = require('log4js').getLogger('(LV)'); const url = require('url'); const removeAccents = require('remove-accents-diacritics'); @@ -12,7 +12,7 @@ class LVScrape extends Scraper { constructor() { super(); - this.id = 'LV'; + this.setID('LV'); this.on('done', () => { this._done(); @@ -57,7 +57,7 @@ class LVScrape extends Scraper { const filename = this.modeNames[this.mode]; - this._makeScreenshotV2(this.page, `${this.path}/${filename}_main_${serviceObject.indexStep}`, null); + this._makeScreenshotV2(this.page, `${this.path}/${filename}_main_${serviceObject.indexStep}`, 1500); this.emit('indexdone'); } @@ -264,7 +264,7 @@ class LVScrape extends Scraper { await this._randomWait(this.page, 3, 5); - await this._makeScreenshotV2(this.page, `${filePath}_main`, null); + await this._makeScreenshotV2(this.page, `${filePath}_main`, 2000); const body = await this.page.content(); @@ -278,10 +278,11 @@ class LVScrape extends Scraper { const qualifyHoldings = await this.extractEntitySubSections(body, 'h2:contains("Qualifying holdings")'); // -- - await jsonfile.writeFile(`${filePath}.json`, { details, marketSegments, relatedPersons, licenses, sanctions, qualifyHoldings }); + + await jsonfile.writeFile(`${filePath}.json`, { details, licenses, marketSegments, qualifyHoldings, relatedPersons, sanctions }); await this._randomWait(this.page, 3, 5); - + if (details.hasOwnProperty('links')) { await this.page._client.send('Page.setDownloadBehavior', { 'behavior': 'allow', 'downloadPath': this.path }); for(const items of details.links) { @@ -293,6 +294,7 @@ class LVScrape extends Scraper { if (!err.message.includes('net::ERR_ABORTED') ) logger.error('grabLink', err); }); + await this._randomWait(this.page, 2, 3); } } diff --git a/ncas/nl.js b/ncas/nl.js index 6905238..6ad8109 100644 --- a/ncas/nl.js +++ b/ncas/nl.js @@ -3,7 +3,7 @@ const cheerio = require('cheerio'); const path = require('path'); const jsonfile = require('jsonfile'); const removeAccents = require('remove-accents-diacritics'); -const logger = require('log4js').getLogger('NL'); +const logger = require('log4js').getLogger('(NL)'); const url = require('url'); logger.level = process.env.LOGGER_LEVEL || 'warn'; @@ -59,7 +59,7 @@ class NLScrape extends Scraper { catch( err) { logger.error(err); } - + return description; } @@ -179,6 +179,7 @@ class NLScrape extends Scraper { const noWhiteSpace = /\W/g; const urlSections = ['WFTBI', 'WFTEG', 'WFTKF']; const id = serviceObject.links[serviceObject.step].id; + const hash = serviceObject.links[serviceObject.step].hash; logger.info(`Process V2 ${this.modeTitles[this.mode]} entity ${serviceObject.step + 1} of ${serviceObject.items} // ${id}`); @@ -188,7 +189,8 @@ class NLScrape extends Scraper { const filename = this._makeFileName(entity); - const filePath = `${this.path}/${filename}`.substring(0, 240); + // DIN-329 - Trim a bit harder and append the unique hash + const filePath = `${`${this.path}/${filename}`.substring(0, 215) }_${hash}`; await this.page.waitForSelector('#contentcolumn > div.interactive-tabs > ol > li:nth-child(2) > a', { 'visible':true, 'timeout':7500 }).then(async (elm) => { await elm.click({ 'delay':Scraper.notARobot() }); @@ -337,7 +339,7 @@ class NLScrape extends Scraper { const options = await this.page.$$('#ddfilter option'); const selects = ['2:12(1) Carrying on the business of a bank', '2:13(1) Carrying on the business of a bank']; const wantedOption = []; - wantedOption.push(selects[this.creditServices.step]); + wantedOption.push(selects[this.creditServices.menuStep]); for (const item of options) { const text = await this.page.evaluate(el => el.innerText, item); const value = await this.page.evaluate(el => el.value, item); @@ -355,6 +357,18 @@ class NLScrape extends Scraper { } } + getQSIdShortHash(inStr) { + const extractor = /([^?=&]+)(=([^&]*))?/g; + + const matched = inStr.match(extractor); + + if (matched.length === 2) + return matched[1].slice(3, 11).toUpperCase(); + + else + return ''; + } + /** * Process WFTBI / Payment Services * @returns {Promise} @@ -366,10 +380,10 @@ class NLScrape extends Scraper { const origUrl = await this.page.url(); const pageUrl = url.parse(origUrl); - if (pageUrl.query === null) - // we need to select the correct item from the dropdown. + if (pageUrl.query === null) + // we need to select the correct item from the dropdown. this.initiateWFTBI(); - + else { // crack query @@ -391,22 +405,26 @@ class NLScrape extends Scraper { statutoryName = removeAccents.remove(statutoryName.trim()).replace(nonWhiteSpace, '_'); - tradeName = removeAccents.remove(tradeName.trim()).replace(nonWhiteSpace, '_'); + tradeName = removeAccents.remove(tradeName.trim()).replace(nonWhiteSpace, '_') || ''; - const id = (statutoryName === tradeName) ? statutoryName : `${statutoryName}-${tradeName}`; + const statutoryNameExists = this.paymentServices.statutoryNames.indexOf(statutoryName.toLowerCase()); + + const id = (statutoryNameExists === -1) ? statutoryName : `${statutoryName}-${tradeName}`; let href = cheerio(children.eq(0)).find('a').attr('href'); - href = href.concat('&locale=en_GB'); - // this is the one we want. + const hash = this.getQSIdShortHash(href); - this.paymentServices.links.push({ id, href }); + href = href.concat('&locale=en_GB'); + + this.paymentServices.links.push({ id, href, hash }); + if (statutoryNameExists === -1) this.paymentServices.statutoryNames.push(statutoryName.toLowerCase()); }); const next = $('a.next').attr('href') || ''; - if (next !== '') + if (next !== '') this._findAndClick('a.next'); - else + else this.emit('startProcessingPaymentServices'); } } @@ -447,16 +465,19 @@ class NLScrape extends Scraper { statutoryName = removeAccents.remove(statutoryName.trim()).replace(nonWhiteSpace, '_'); - tradeName = removeAccents.remove(tradeName.trim()).replace(nonWhiteSpace, '_'); + tradeName = removeAccents.remove(tradeName.trim()).replace(nonWhiteSpace, '_') || ''; - // const id = `${statutoryName}-${tradeName}`; - const id = (statutoryName === tradeName) ? statutoryName : `${statutoryName}-${tradeName}`; + const statutoryNameExists = this.emoneyServices.statutoryNames.indexOf(statutoryName.toLowerCase()); + + const id = (statutoryNameExists === -1) ? statutoryName : `${statutoryName}-${tradeName}`; let href = cheerio(children.eq(0)).find('a').attr('href'); - href = href.concat('&locale=en_GB'); - // this is the one we want. + const hash = this.getQSIdShortHash(href); - this.emoneyServices.links.push({ id, href }); + href = href.concat('&locale=en_GB'); + + this.emoneyServices.links.push({ id, href, hash }); + if (statutoryNameExists === -1) this.emoneyServices.statutoryNames.push(statutoryName.toLowerCase()); }); const next = $('a.next').attr('href') || ''; @@ -506,19 +527,19 @@ class NLScrape extends Scraper { statutoryName = removeAccents.remove(statutoryName.trim()).replace(nonWhiteSpace, '_'); - tradeName = removeAccents.remove(tradeName.trim()).replace(nonWhiteSpace, '_'); + tradeName = removeAccents.remove(tradeName.trim()).replace(nonWhiteSpace, '_') || ''; - const id = (statutoryName === tradeName) ? statutoryName : `${statutoryName}-${tradeName}`; + const statutoryNameExists = this.creditServices.statutoryNames.indexOf(statutoryName.toLowerCase()); - // const id = `${statutoryName}-${tradeName}`; + const id = (statutoryNameExists === -1) ? statutoryName : `${statutoryName}-${tradeName}`; let href = cheerio(children.eq(0)).find('a').attr('href'); + const hash = this.getQSIdShortHash(href); + href = href.concat('&locale=en_GB'); - // this is the one we want. - logger.debug({ id, href }); - - this.creditServices.links.push({ id, href }); + this.creditServices.links.push({ id, href, hash }); + if (statutoryNameExists === -1) this.creditServices.statutoryNames.push(statutoryName.toLowerCase()); }); const next = $('a.next').attr('href') || ''; @@ -526,8 +547,8 @@ class NLScrape extends Scraper { if (next !== '') this._findAndClick('a.next'); else - if (this.creditServices.step === 0) { - this.creditServices.step = 1; + if (this.creditServices.menuStep === 0) { + this.creditServices.menuStep = 1; await this._goto(this.credit); } else @@ -591,24 +612,32 @@ class NLScrape extends Scraper { * @returns {Promise} */ async restart() { + // v2 logger.info(`Restarting ${this.modeTitles[this.mode]}`); - switch (this.mode) { - - case 2: - this.emit('startProcessingCreditServices'); - break; - - case 1: - this.emit('startProcessingEMoneyServices'); - break; - - case 0: - default: + if (this.mode === 0) + if (this.paymentServices.links.length > 0) { this.emit('startProcessingPaymentServices'); - break; - - } + } + else { + await this._goto(this.startPage); + } + + else if(this.mode === 1) + if (this.emoneyServices.links.length > 0) { + this.emit('startProcessingEMoneyServices'); + } + else { + await this._goto(this.emoneyUrl); + } + + else if(this.mode === 2) + if (this.creditServices.links.length > 0) { + this.emit('startProcessingCreditServices'); + } + else { + await this._goto(this.credit); + } } /** @@ -626,12 +655,13 @@ class NLScrape extends Scraper { this.processNewPage(); }); - const timeout = 90000; + const antiCollision = 125 + (Math.floor(Math.random() * (15 - 1)) * 500); + const timeout = 90000 + antiCollision; - setTimeout(async() => { + setTimeout(() => { logger.warn('Attempting recovery..'); - await this.restart(); + this.restart(); }, timeout); } @@ -726,29 +756,33 @@ class NLScrape extends Scraper { this.mode = 0; try { this.paymentServices = { + 'done' : false, 'items': 0, 'links': [], + 'statutoryNames': [], 'step': 0, - 'visited': false, - 'done' : false + 'visited': false }; this.emoneyServices = { + 'done' : false, 'items': 0, 'links': [], + 'searchDone' : false, + 'statutoryNames': [], 'step': 0, - 'visited': false, - 'done' : false, - 'searchDone' : false + 'visited': false }; this.creditServices = { + 'done' : false, 'items': 0, 'links': [], + 'menuStep' : 0, + 'searchDone' : false, + 'statutoryNames': [], 'step': 0, - 'visited': false, - 'done' : false, - 'searchDone' : false + 'visited': false }; this.startPage = 'https://www.dnb.nl/en/supervision/public-register/WFTBI/index.jsp'; diff --git a/ncas/no.js b/ncas/no.js index 86a33a9..0a564a7 100644 --- a/ncas/no.js +++ b/ncas/no.js @@ -4,7 +4,7 @@ const Scraper = require('../helpers/scraper'); const cheerio = require('cheerio'); const path = require('path'); const jsonfile = require('jsonfile'); -const logger = require('log4js').getLogger('NO'); +const logger = require('log4js').getLogger('(NO)'); const url = require('url'); const removeAccents = require('remove-accents-diacritics'); @@ -14,7 +14,15 @@ class NOScrape extends Scraper { constructor() { super(); - this.id = 'NO'; + this.setID('NO'); + + // treat these elements as block boundaries when scraping crossborder permissions + this.blockBoundaries = 'div, li'; + + // override these values from the base class + this.modePrefix = ['ps_', 'ps__in_', 'em_', 'em__in_', 'ci_', 'ci__in_']; + this.modeNames = ['paymentServices', 'paymentServicesIncoming', 'emoneyServices', 'emoneyServicesIncoming', 'creditServices', 'creditServicesIncoming']; + this.modeTitles = ['Payment Service', 'Payment Service Incoming', 'EMoney', 'EMoney Incoming', 'Credit Services', 'Credit Services Incoming']; this.on('done', () => { this._done(); @@ -31,6 +39,70 @@ class NOScrape extends Scraper { }); } + _reduceWhiteSpace(text) { + return text.replace(/\s+/g, ' ').trim(); + } + + /** + * + * @param html + * @param selector + * @returns {Promise} + * + * Finds elements in the `html` with the given `selector`, but returns only the uppermost matched elements, + * and not those that are nested within other matched elements. + */ + getUppermostElementsBySelector(html, selector) { + const $ = cheerio.load(html); + + return $(selector).filter(function () { + return $(this).parents(selector).length === 0; + }); + } + + getTextNotInMatchingElements(html, selector) { + const $ = cheerio.load(html); + + $(selector) + .remove() + .end(); + + return $.text(); + } + + recurseDOM(html, selector, level = 0) { + const currentLevel = level + 1; + const $ = cheerio.load(html); + + const result = []; + + const blocks = this.getUppermostElementsBySelector(html, selector); + + for (let i = 0; i < blocks.length; i++) { + const block = blocks[i]; + + const rawName = this.getTextNotInMatchingElements($(block).html(), selector); + const name = this._reduceWhiteSpace(rawName); + + const blockHtml = $(block).html(); + + const data = this.recurseDOM(blockHtml, selector, currentLevel); + + if (data === null) + result.push(name); + else + result.push({ + 'name': name, + 'data': data + }); + } + + if (result.length > 0) + return result; + + return null; + } + /** * * @param html @@ -184,6 +256,23 @@ class NOScrape extends Scraper { return newObj; } + recurseCrossborderHtml(html) { + try { + // Wrap html in an outer div to give the recursion loop a root starting point + const wrappedHtml = `
${html}
`; + + // recurse the dom, using the selectors we have defined as block boundaries + const results = this.recurseDOM(wrappedHtml, this.blockBoundaries); + + // return the first item from our list (i.e. the outer div we added at the top of this function) + return results[0]; + } + catch(err) { + logger.error('Error during recursion of cross-border HTML'); + logger.error(err); + } + } + /** * * @param html @@ -191,40 +280,15 @@ class NOScrape extends Scraper { */ extractEntityDetailCrossBorder(html) { try { - const newObj = { }; - const $ = cheerio.load(html); const header = $('h3.license-unit-label:contains("Cross-border services/classes")'); const detailBox = $(header).parent(); - const children = $(detailBox).children(); + const crossborder = this.recurseCrossborderHtml(detailBox.html()); - let curLabel = ''; - children.each(async (i, item) => { - const tagName = $(item).prop('tagName'); - - if (tagName === 'H3') { - curLabel = this._makeFieldName($(item).text()); - if (!newObj.hasOwnProperty(curLabel)) - newObj[curLabel] = []; - } - - if (['SPAN', 'A', 'P'].indexOf(tagName) !== -1) - newObj[curLabel].push(this._cleanUp($(item).text())); - - if(tagName === 'DIV' || tagName === 'UL') { - if (!newObj.hasOwnProperty('data')) - newObj['data'] = []; - - const cbData = this.extractCrossBorderDetailsV2($(item).html()); - - newObj['data'].push(cbData); - } - }); - - return newObj; + return crossborder; } catch( err) { logger.error(err); @@ -254,6 +318,17 @@ class NOScrape extends Scraper { } } + /** + * + * @param serviceObject + * @param elm + * @returns {Promise} + */ + async selectCountryOption(serviceObject, elm) { + const countryOption = serviceObject.country; + await this.page.select('select.search-filter[aria-label="Choose country"]', countryOption); + } + /** * * @param html @@ -322,7 +397,7 @@ class NOScrape extends Scraper { await this._randomWait(this.page, 5, 7); - this._makeScreenshotV2(this.page, `${this.path}/${filename}_main_${serviceObject.indexStep}`, null); + this._makeScreenshotV2(this.page, `${this.path}/${filename}_main_${serviceObject.indexStep}`, 2500); this.emit('indexdone'); } @@ -346,6 +421,18 @@ class NOScrape extends Scraper { logger.warn('Waiting for data timeout'); }); + await this._randomWait(this.page, 3, 5); + + await this.page.waitForSelector('select.search-filter[aria-label="Choose country"]', { 'visible':true, 'timeout':7500 }).then(async (elm) => { + await this.selectCountryOption(serviceObject, elm); + }).catch((e) => { + logger.error(e); + logger.warn('No country select'); + }); + + // Give the country selection some time to filter the results + await this._randomWait(this.page, 3, 5); + await this.page.waitForSelector('#js-konsregList > div > div', { 'visible':true, 'timeout':7500 }).then(async (elm) => { await this.processIndex(serviceObject); }).catch((e) => { @@ -375,7 +462,8 @@ class NOScrape extends Scraper { { 'find':'h2:contains("Agency debt collection on behalf of others")', 'blockType':'Debt collection' }, { 'find':'h2:contains("E-money institution")', 'blockType':'E-money institution' }, { 'find':'h2:contains("Investment firm")', 'blockType':'h2:contains("Investment firm")' }, - { 'find':'h2:contains("Intermediator of loans and guarantees")', 'blockType':'Intermediator of loans and guarantees' } + { 'find':'h2:contains("Intermediator of loans and guarantees")', 'blockType':'Intermediator of loans and guarantees' }, + { 'find':'h2:contains("Payment service provider with a limited authorisat")', 'blockType':'Payment service provider with a limited authorisat' } ]; @@ -421,7 +509,7 @@ class NOScrape extends Scraper { await this._randomWait(this.page, 5, 7); - await this._makeScreenshotV2(this.page, `${filePath}_main`, null); + await this._makeScreenshotV2(this.page, `${filePath}_main`, 2500); const body = await this.page.content(); @@ -460,13 +548,25 @@ class NOScrape extends Scraper { break; case 1: - await this.buildIndex(this.emoneyServices); + await this.buildIndex(this.paymentServicesIncoming); break; case 2: + await this.buildIndex(this.emoneyServices); + break; + + case 3: + await this.buildIndex(this.emoneyServicesIncoming); + break; + + case 4: await this.buildIndex(this.creditServices); break; + case 5: + await this.buildIndex(this.creditServicesIncoming); + break; + } } @@ -482,13 +582,25 @@ class NOScrape extends Scraper { break; case 1: - await this.processEntityDetails(this.emoneyServices); + await this.processEntityDetails(this.paymentServicesIncoming); break; case 2: + await this.processEntityDetails(this.emoneyServices); + break; + + case 3: + await this.processEntityDetails(this.emoneyServicesIncoming); + break; + + case 4: await this.processEntityDetails(this.creditServices); break; + case 5: + await this.processEntityDetails(this.creditServicesIncoming); + break; + } } @@ -553,13 +665,25 @@ class NOScrape extends Scraper { break; case 1: - this.emit('emoneyServicesDone'); + this.emit('paymentServicesIncomingDone'); break; case 2: + this.emit('emoneyServicesDone'); + break; + + case 3: + this.emit('emoneyServicesIncomingDone'); + break; + + case 4: this.emit('creditServicesDone'); break; + case 5: + this.emit('creditServicesIncomingDone'); + break; + } }); @@ -578,6 +702,21 @@ class NOScrape extends Scraper { await this._goto(newUrl); }); + this.on('psincindexdone', async () => { + let newUrl; + this.paymentServicesIncoming.items = this.paymentServicesIncoming.links.length; + logger.info(`${this.paymentServicesIncoming.items} items indexed`); + + this.paymentServicesIncoming.indexStep++; + if (this.paymentServicesIncoming.indexStep >= this.paymentServicesIncoming.wanted.length) + newUrl = this.paymentServicesIncoming.links[this.paymentServicesIncoming.step].href; + + else + newUrl = this.paymentServicesIncoming.urls[0]; + + await this._goto(newUrl); + }); + this.on('emindexdone', async () => { let newUrl; this.emoneyServices.items = this.emoneyServices.links.length; @@ -592,6 +731,20 @@ class NOScrape extends Scraper { await this._goto(newUrl); }); + this.on('emincindexdone', async () => { + let newUrl; + this.emoneyServicesIncoming.items = this.emoneyServicesIncoming.links.length; + logger.info(`${this.emoneyServicesIncoming.items} items indexed`); + + this.emoneyServicesIncoming.indexStep++; + if (this.emoneyServicesIncoming.indexStep >= this.emoneyServicesIncoming.urls.length) + newUrl = this.emoneyServicesIncoming.links[this.emoneyServicesIncoming.step].href; + else + newUrl = this.emoneyServicesIncoming.urls[this.emoneyServicesIncoming.indexStep]; + + await this._goto(newUrl); + }); + this.on('ciindexdone', async () => { let newUrl; this.creditServices.items = this.creditServices.links.length; @@ -605,6 +758,20 @@ class NOScrape extends Scraper { await this._goto(newUrl); }); + + this.on('ciincindexdone', async () => { + let newUrl; + this.creditServicesIncoming.items = this.creditServicesIncoming.links.length; + logger.info(`${this.creditServicesIncoming.items} items indexed`); + + this.creditServicesIncoming.indexStep++; + if (this.creditServicesIncoming.indexStep >= this.creditServicesIncoming.urls.length) + newUrl = this.creditServicesIncoming.links[this.creditServicesIncoming.step].href; + else + newUrl = this.creditServicesIncoming.urls[this.creditServicesIncoming.indexStep]; + + await this._goto(newUrl); + }); this.on('indexdone', async () => { switch (this.mode) { @@ -614,13 +781,25 @@ class NOScrape extends Scraper { break; case 1: - this.emit('emindexdone'); + this.emit('psincindexdone'); break; case 2: + this.emit('emindexdone'); + break; + + case 3: + this.emit('emincindexdone'); + break; + + case 4: this.emit('ciindexdone'); break; + case 5: + this.emit('ciincindexdone'); + break; + } }); @@ -634,7 +813,24 @@ class NOScrape extends Scraper { this.mode++; this.inProgress = false; - await this._goto(this.emoneyServices.urls[0]); + await this._goto(this.paymentServicesIncoming.urls[0]); + } + catch (e) { + logger.error(e); + } + }); + + this.on('paymentServicesIncomingDone', async () => { + logger.warn('paymentServicesIncomingDone'); + try{ + this.paymentServicesIncoming.done = true; + jsonfile.writeFileSync(`${this.path}/paymentServicesIncoming.json`, { 'links': this.paymentServicesIncoming.links }); + jsonfile.writeFileSync(`${this.debugPath}/paymentServicesIncoming.json`, this.paymentServicesIncoming); + + this.mode++; + this.inProgress = false; + + await this._goto(this.emoneyServicesIncoming.urls[0]); } catch (e) { logger.error(e); @@ -650,13 +846,29 @@ class NOScrape extends Scraper { this.mode++; this.inProgress = false; - await this._goto(this.creditServices.urls[0]); + await this._goto(this.emoneyServicesIncoming.urls[0]); } catch (e) { logger.error(e); } }); + this.on('emoneyServicesIncomingDone', async () => { + logger.warn('emoneyServicesIncomingDone'); + try{ + this.emoneyServicesIncoming.done = true; + jsonfile.writeFileSync(`${this.path}/emoneyServicesIncoming.json`, { 'links':this.emoneyServicesIncoming.links }); + jsonfile.writeFileSync(`${this.debugPath}/emoneyServicesIncoming.json`, this.emoneyServicesIncoming); + this.mode++; + this.inProgress = false; + + await this._goto(this.creditServices.urls[0]); + } + catch (e) { + logger.error(e); + } + }); + this.on('creditServicesDone', async () => { logger.warn('creditServicesDone'); try{ @@ -666,6 +878,22 @@ class NOScrape extends Scraper { this.mode++; this.inProgress = false; + await this._goto(this.creditServicesIncoming.urls[0]); + } + catch (e) { + logger.error(e); + } + }); + + this.on('creditServicesIncomingDone', async () => { + logger.warn('creditServicesIncomingDone'); + try{ + this.creditServicesIncoming.done = true; + jsonfile.writeFileSync(`${this.path}/creditServicesIncoming.json`, { 'links':this.creditServicesIncoming.links }); + jsonfile.writeFileSync(`${this.debugPath}/creditServicesIncoming.json`, this.creditServicesIncoming); + this.mode++; + this.inProgress = false; + this.emit('done'); } catch (e) { @@ -694,6 +922,21 @@ class NOScrape extends Scraper { 'done' : false, 'urls': ['https://www.finanstilsynet.no/en/finanstilsynets-registry/'], 'wanted' : ['Payment institution', 'Agent of payment institution (company)', 'Payment service provider with a limited authorisat'], + 'country' : 'NOR', + 'sections' : [], + 'sectionLinks' : [] + }; + + this.paymentServicesIncoming = { + 'items': 0, + 'links': [], + 'step': 0, + 'indexStep': 0, + 'visited': false, + 'done' : false, + 'urls': ['https://www.finanstilsynet.no/en/finanstilsynets-registry/'], + 'wanted' : ['Payment institution', 'Agent of payment institution (company)', 'Payment service provider with a limited authorisat'], + 'country' : 'NOTNOR', 'sections' : [], 'sectionLinks' : [] }; @@ -707,6 +950,21 @@ class NOScrape extends Scraper { 'done' : false, 'urls': ['https://www.finanstilsynet.no/en/finanstilsynets-registry/'], 'wanted' : ['E-money institution'], + 'country' : 'NOR', + 'sections' : [], + 'sectionLinks' : [] + }; + + this.emoneyServicesIncoming = { + 'items': 0, + 'links': [], + 'step': 0, + 'indexStep': 0, + 'visited': false, + 'done' : false, + 'urls': ['https://www.finanstilsynet.no/en/finanstilsynets-registry/'], + 'wanted' : ['E-money institution'], + 'country' : 'NOTNOR', 'sections' : [], 'sectionLinks' : [] }; @@ -722,13 +980,28 @@ class NOScrape extends Scraper { 'started': false, 'urls': ['https://www.finanstilsynet.no/en/finanstilsynets-registry/'], 'wanted' : ['Bank', 'Branch of foreign credit institution', 'Credit Institution', 'Savings bank foundation'], + 'country' : 'NOR', + 'sections' : [], + 'sectionLinks' : [] + }; + + this.creditServicesIncoming = { + 'items': 0, + 'links': [], + 'step': 0, + 'indexStep': 0, + 'visited': false, + 'done' : false, + 'searchDone' : false, + 'started': false, + 'urls': ['https://www.finanstilsynet.no/en/finanstilsynets-registry/'], + 'wanted' : ['Bank', 'Branch of foreign credit institution', 'Credit Institution', 'Savings bank foundation'], + 'country' : 'NOTNOR', 'sections' : [], 'sectionLinks' : [] }; this.startPage = this.paymentServices.urls[0]; - this.emoneyUrl = this.emoneyServices.urls[0]; - this.credit = this.creditServices.urls[0]; this.setPath(path.resolve(`${__dirname }/../artefacts/NO/FS`)); diff --git a/ncas/pl.js b/ncas/pl.js index 02dd948..fdff60c 100644 --- a/ncas/pl.js +++ b/ncas/pl.js @@ -704,7 +704,7 @@ class PLScrape extends Scraper { } catch( err) { logger.error(err); - this.emit('recover'); + this.emit('backoff'); } } @@ -926,7 +926,10 @@ class PLScrape extends Scraper { }); if (doIndex) - await this.processEntityIndex(this.paymentServices); + await this.processEntityIndex(this.paymentServices).catch(async (err) => { + logger.error('processEntityIndex catch: ', err); + this.emit('restart'); + }); } /** @@ -1021,7 +1024,7 @@ class PLScrape extends Scraper { if (pageUrl.href === 'chrome-error://chromewebdata/') { logger.warn('Directed to: chrome-error://chromewebdata/'); - this.emit('recover'); + this.emit('backoff'); return; } @@ -1102,7 +1105,7 @@ class PLScrape extends Scraper { * * @returns {Promise} */ - async restart() { + async _restart() { logger.warn(`Tryng to restart ${this.modeTitles[this.mode]}`); if (this.mode === 0) { @@ -1145,9 +1148,10 @@ class PLScrape extends Scraper { if (this.backOffStep > this.backOffLimit) this.backOffStep = this.backOffLimit; logger.warn(`Backing off for ${this.backOffStep * 5} minutes..`); - const timeout = (60 * 1000) * (this.backOffStep * 5); + const timeout = 300000; // (this.backOffStep * 5) * 60000; - await this._uploadError(); + logger.warn('timeout', timeout); + // await this._uploadError(); this.backOffTimer = setTimeout(() => { this.emit('restart'); @@ -1166,15 +1170,19 @@ class PLScrape extends Scraper { }, 2500)); this.on('stall', () => { - this.backoff(); + this.emit('backoff'); }); - this.on('backoff', () => { + this.on('backoff', this._debounce( () => { this.backoff(); - }); + }, 10000)); + + /* this.on('backoff', () => { + this.backoff(); + });*/ this.on('restart', async () => { - await this.restart(); + await this._restart(); }); this.on('entityComplete', () => { diff --git a/ncas/pt.js b/ncas/pt.js index 973bc71..9ae2496 100644 --- a/ncas/pt.js +++ b/ncas/pt.js @@ -264,13 +264,15 @@ class PTScrape extends Scraper { const pageUrl = url.parse(await this.page.url()); - if (pageUrl.href === 'chrome-error://chromewebdata/') { - logger.warn('Directed to: chrome-error://chromewebdata/'); + if (pageUrl.href === 'chrome-error://chromewebdata/' || pageUrl.pathname === null) { + logger.warn('Directed to: chrome-error://chromewebdata/ or null'); this.emit('recover'); return; } + logger.debug('processNewPage', pageUrl.pathname); + const splitPath = pageUrl.pathname.match(pathSplitter); const pathname = splitPath[0]; diff --git a/ncas/sk.js b/ncas/sk.js index 0257769..877a1a5 100644 --- a/ncas/sk.js +++ b/ncas/sk.js @@ -168,62 +168,68 @@ class SKScrape extends Scraper { * @returns {Promise} */ async processEntityIndex(serviceObject) { - const fields = ['referenceNumber', 'businessName', 'address', 'start', 'end', 'reason']; + try{ + const fields = ['referenceNumber', 'businessName', 'address', 'start', 'end', 'reason']; - const mouseDownDuration = Scraper.notARobot(); - if (serviceObject.visited === false) { - logger.debug('Preparing...'); + const mouseDownDuration = Scraper.notARobot(); + if (serviceObject.visited === false) { + logger.debug('Preparing...'); - await this.page.waitForSelector('table#Subjects', { 'visible':true }).then(async () => { - await this.entityIndexFirstPass(serviceObject); - }).catch(() => { - logger.error('Table failed to render'); - }); - } + await this.page.waitForSelector('table#Subjects', { 'visible':true }).then(async () => { + await this.entityIndexFirstPass(serviceObject); + }).catch(() => { + logger.error('Table failed to render'); + }); + } - if (serviceObject.visited === true) { - serviceObject.currentMetaIndex = serviceObject.step % 10; + if (serviceObject.visited === true) { + serviceObject.currentMetaIndex = serviceObject.step % 10; - if ((serviceObject.step ) >= serviceObject.currentPageMax) { - const nextButton = await this.page.$$('#Subjects_next'); + if ((serviceObject.step ) >= serviceObject.currentPageMax) { + const nextButton = await this.page.$$('#Subjects_next'); - const buttonClasses = await this.page.$eval('#Subjects_next', e => e.getAttribute('class')); + const buttonClasses = await this.page.$eval('#Subjects_next', e => e.getAttribute('class')); - if (buttonClasses.split(' ').indexOf('disabled') === -1) { - // we need a click.. - nextButton[0].click({ 'delay':mouseDownDuration }); + if (buttonClasses.split(' ').indexOf('disabled') === -1) { + // we need a click.. + nextButton[0].click({ 'delay':mouseDownDuration }); + + await this._randomWait(this.page, 3, 5); + + serviceObject.visited = false; + this.emit('entityIndex'); + } + else { + logger.debug('I think we are done here...'); + this.emit('serviceDone'); + } + } + + else { + await this.page.waitForSelector('#Subjects > tbody'); + + const wantedRow = await this.page.$$(`#Subjects > tbody > tr:nth-child(${serviceObject.currentMetaIndex + 1})`); + const htmlRow = await this.page.evaluate(el => el.outerHTML, wantedRow[0]); + + const $ = cheerio.load(`${htmlRow}
`); + + const cells = $('td'); + + serviceObject.current = {}; + + cells.each((index, item) => { + serviceObject.current[ fields[index] ] = $(item).text(); + }); await this._randomWait(this.page, 3, 5); - serviceObject.visited = false; - this.emit('entityIndex'); - } - else { - logger.debug('I think we are done here...'); - this.emit('serviceDone'); + await wantedRow[0].click({ 'delay':mouseDownDuration }); } } - - else { - await this.page.waitForSelector('#Subjects > tbody'); - - const wantedRow = await this.page.$$(`#Subjects > tbody > tr:nth-child(${serviceObject.currentMetaIndex + 1})`); - const htmlRow = await this.page.evaluate(el => el.outerHTML, wantedRow[0]); - - const $ = cheerio.load(`${htmlRow}
`); - - const cells = $('td'); - - serviceObject.current = {}; - - cells.each((index, item) => { - serviceObject.current[ fields[index] ] = $(item).text(); - }); - - await this._randomWait(this.page, 3, 5); - - await wantedRow[0].click({ 'delay':mouseDownDuration }); - } + } + catch( err) { + logger.error(err); + this.emit('recover'); } } @@ -386,76 +392,82 @@ class SKScrape extends Scraper { * @returns {Promise} */ async processEntityDetail(serviceObject) { - // level0 sublicctrl sublicctrl1 odd - // level0 sublicctrl sublicctrl1 odd sublicshow shown + try{ + // level0 sublicctrl sublicctrl1 odd + // level0 sublicctrl sublicctrl1 odd sublicshow shown - // expand all accordians + // expand all accordians - const rows = await this.page.$$('tr.sublicctrl'); + const rows = await this.page.$$('tr.sublicctrl'); - for (const item of rows) { - const cls = await this.page.evaluate(el => el.getAttribute('class'), item); - if (!cls.includes('shown')) + for (const item of rows) { + const cls = await this.page.evaluate(el => el.getAttribute('class'), item); + if (!cls.includes('shown')) - await item.click({ 'delay':Scraper.notARobot() }); - } - - await this.page.waitForSelector('#Licenses > tbody > tr.level1.shown.sublichide1.sllhidectrl.sllhidectrl1', { 'timeout':7500 }).then(async (elm) => { - await elm.click({ 'delay':Scraper.notARobot() }); - }).catch(() => { - logger.debug('No License information'); - }); - - await this._microWait(this.page, 5); - - // expand all viewable anchors - const wantedAnchors = await this.page.$$('.row a'); - - for (const item of wantedAnchors) { - const exItem = this._cleanUp(await this.page.evaluate(el => el.text, item)); - - if (exItem === 'View') { - await item.hover().catch((e) => { - logger.warn('Hover failed', e.name); - }); - - await item.click({ 'delay': Scraper.notARobot() }).catch((e) => { - logger.debug('View click failed', e.name); - }); + await item.click({ 'delay':Scraper.notARobot() }); } + + await this.page.waitForSelector('#Licenses > tbody > tr.level1.shown.sublichide1.sllhidectrl.sllhidectrl1', { 'timeout':7500 }).then(async (elm) => { + await elm.click({ 'delay':Scraper.notARobot() }); + }).catch(() => { + logger.debug('No License information'); + }); + + await this._microWait(this.page, 5); + + // expand all viewable anchors + const wantedAnchors = await this.page.$$('.row a'); + + for (const item of wantedAnchors) { + const exItem = this._cleanUp(await this.page.evaluate(el => el.text, item)); + + if (exItem === 'View') { + await item.hover().catch((e) => { + logger.warn('Hover failed', e.name); + }); + + await item.click({ 'delay': Scraper.notARobot() }).catch((e) => { + logger.debug('View click failed', e.name); + }); + } + } + + const entityName = `${serviceObject.current.businessName}_${serviceObject.current.referenceNumber}`; + const fileName = this._makeFileName(entityName); + const filePath = await this._makeFilePath(entityName); + + serviceObject.current.fileName = fileName; + + await this._randomWait(this.page, 2, 2); + await this.page.focus('h3.page-header'); + await this._makeScreenshotV2(this.page, `${filePath}_main`, null); + + await this.page.waitForSelector('body > div.container > form.form-horizontal > table', { 'timeout':7500 }).then(async (elm) => { + logger.debug('prep for processEntityDetailBasicDetails'); + + const htmlBlock = await this.page.evaluate(el => el.outerHTML, elm); + + const $ = cheerio.load(htmlBlock); + + serviceObject.current.basicDetails = await this.processEntityDetailBasicDetails($); + }); + + await this.page.waitForSelector('#Licenses').then(async (elm) => { + logger.debug('prep for processEntityDetailTableV2'); + + const htmlBlock = await this.page.evaluate(el => el.outerHTML, elm); + + const $ = cheerio.load(htmlBlock); + + serviceObject.current.entityDetails = await this.processEntityDetailTableV2($); + }); + + this.entityCompleter(serviceObject); + } + catch( err) { + logger.error(err); + this.emit('recover'); } - - const entityName = `${serviceObject.current.businessName}_${serviceObject.current.referenceNumber}`; - const fileName = this._makeFileName(entityName); - const filePath = await this._makeFilePath(entityName); - - serviceObject.current.fileName = fileName; - - await this._randomWait(this.page, 2, 2); - await this.page.focus('h3.page-header'); - await this._makeScreenshotV2(this.page, `${filePath}_main`, null); - - await this.page.waitForSelector('body > div.container > form.form-horizontal > table', { 'timeout':7500 }).then(async (elm) => { - logger.debug('prep for processEntityDetailBasicDetails'); - - const htmlBlock = await this.page.evaluate(el => el.outerHTML, elm); - - const $ = cheerio.load(htmlBlock); - - serviceObject.current.basicDetails = await this.processEntityDetailBasicDetails($); - }); - - await this.page.waitForSelector('#Licenses').then(async (elm) => { - logger.debug('prep for processEntityDetailTableV2'); - - const htmlBlock = await this.page.evaluate(el => el.outerHTML, elm); - - const $ = cheerio.load(htmlBlock); - - serviceObject.current.entityDetails = await this.processEntityDetailTableV2($); - }); - - this.entityCompleter(serviceObject); } /** diff --git a/package-lock.json b/package-lock.json index 8f134a0..a438c75 100644 --- a/package-lock.json +++ b/package-lock.json @@ -353,7 +353,7 @@ }, "ansi-colors": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-1.1.0.tgz", + "resolved": "http://registry.npmjs.org/ansi-colors/-/ansi-colors-1.1.0.tgz", "integrity": "sha512-SFKX67auSNoVR38N3L+nvsPjOE0bybKTYbkf5tRvushrAPQ9V75huw0ZxBkKVeRU9kqH3d6HA4xTckbwZ4ixmA==", "dev": true, "requires": { @@ -1253,7 +1253,7 @@ "dependencies": { "uuid": { "version": "2.0.3", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz", + "resolved": "http://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz", "integrity": "sha1-Z+LoY3lyFVMN/zGOW/nc6/1Hsho=" } } @@ -1690,7 +1690,7 @@ }, "duplexer": { "version": "0.1.1", - "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz", + "resolved": "http://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz", "integrity": "sha1-rOb/gIwc5mtX0ev5eXessCM0z8E=", "dev": true }, @@ -1711,7 +1711,7 @@ }, "readable-stream": { "version": "1.1.14", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", "dev": true, "requires": { @@ -2816,12 +2816,6 @@ "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz", "integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w==" }, - "get-own-enumerable-property-symbols": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.0.tgz", - "integrity": "sha512-CIJYJC4GGF06TakLg8z4GQKvDsx9EMspVxOYih7LerEL/WosUnFIww45CGfxfeKHqlg3twgUrYRT1O3WQqjGCg==", - "dev": true - }, "get-ssl-certificate": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/get-ssl-certificate/-/get-ssl-certificate-2.3.1.tgz", @@ -2979,7 +2973,7 @@ }, "readable-stream": { "version": "1.0.34", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", "dev": true, "requires": { @@ -3103,7 +3097,7 @@ }, "lodash": { "version": "1.0.2", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-1.0.2.tgz", + "resolved": "http://registry.npmjs.org/lodash/-/lodash-1.0.2.tgz", "integrity": "sha1-j1dWDIO1n8JwvT1WG2kAQ0MOJVE=", "dev": true }, @@ -3156,7 +3150,7 @@ }, "got": { "version": "6.7.1", - "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz", + "resolved": "http://registry.npmjs.org/got/-/got-6.7.1.tgz", "integrity": "sha1-JAzQV4WpoY5WHcG0S0HHY+8ejbA=", "requires": { "create-error-class": "^3.0.0", @@ -3179,7 +3173,7 @@ }, "gulp": { "version": "3.9.1", - "resolved": "https://registry.npmjs.org/gulp/-/gulp-3.9.1.tgz", + "resolved": "http://registry.npmjs.org/gulp/-/gulp-3.9.1.tgz", "integrity": "sha1-VxzkWSjdQK9lFPxAEYZgFsE4RbQ=", "dev": true, "requires": { @@ -3212,7 +3206,7 @@ }, "chalk": { "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", "dev": true, "requires": { @@ -3225,19 +3219,19 @@ }, "minimist": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", "dev": true }, "semver": { "version": "4.3.6", - "resolved": "https://registry.npmjs.org/semver/-/semver-4.3.6.tgz", + "resolved": "http://registry.npmjs.org/semver/-/semver-4.3.6.tgz", "integrity": "sha1-MAvG4OhjdPe6YQaLWx7NV/xlMto=", "dev": true }, "strip-ansi": { "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "dev": true, "requires": { @@ -3282,13 +3276,13 @@ }, "async": { "version": "0.9.2", - "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz", + "resolved": "http://registry.npmjs.org/async/-/async-0.9.2.tgz", "integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=", "dev": true }, "bl": { "version": "0.9.5", - "resolved": "https://registry.npmjs.org/bl/-/bl-0.9.5.tgz", + "resolved": "http://registry.npmjs.org/bl/-/bl-0.9.5.tgz", "integrity": "sha1-wGt5evCF6gC8Unr8jvzxHeIjIFQ=", "dev": true, "requires": { @@ -3346,7 +3340,7 @@ }, "lodash": { "version": "3.2.0", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.2.0.tgz", + "resolved": "http://registry.npmjs.org/lodash/-/lodash-3.2.0.tgz", "integrity": "sha1-S/UKMkP5rrC6xBpV09WZBnWkYvs=", "dev": true }, @@ -3361,7 +3355,7 @@ }, "readable-stream": { "version": "1.0.34", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", "dev": true, "requires": { @@ -3379,7 +3373,7 @@ }, "tar-stream": { "version": "1.1.5", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.1.5.tgz", + "resolved": "http://registry.npmjs.org/tar-stream/-/tar-stream-1.1.5.tgz", "integrity": "sha1-vpIYwTDCACnhB7D5Z/sj3gV50Tw=", "dev": true, "requires": { @@ -3461,29 +3455,6 @@ "through2": "^2.0.1" } }, - "gulp-changed-in-place": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/gulp-changed-in-place/-/gulp-changed-in-place-2.3.0.tgz", - "integrity": "sha1-3kFQwnbYwAUkUcry8MfSvhfnaIg=", - "dev": true, - "requires": { - "through2": "^2.0.0" - } - }, - "gulp-debug": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/gulp-debug/-/gulp-debug-4.0.0.tgz", - "integrity": "sha512-cn/GhMD2nVZCVxAl5vWao4/dcoZ8wUJ8w3oqTvQaGDmC1vT7swNOEbhQTWJp+/otKePT64aENcqAQXDcdj5H1g==", - "dev": true, - "requires": { - "chalk": "^2.3.0", - "fancy-log": "^1.3.2", - "plur": "^3.0.0", - "stringify-object": "^3.0.0", - "through2": "^2.0.0", - "tildify": "^1.1.2" - } - }, "gulp-gzip": { "version": "1.4.2", "resolved": "https://registry.npmjs.org/gulp-gzip/-/gulp-gzip-1.4.2.tgz", @@ -3651,7 +3622,7 @@ }, "chalk": { "version": "1.1.3", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", "dev": true, "requires": { @@ -3670,7 +3641,7 @@ }, "minimist": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", + "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", "dev": true }, @@ -3682,7 +3653,7 @@ }, "strip-ansi": { "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "dev": true, "requires": { @@ -3974,12 +3945,6 @@ "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=" }, - "irregular-plurals": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/irregular-plurals/-/irregular-plurals-2.0.0.tgz", - "integrity": "sha512-Y75zBYLkh0lJ9qxeHlMjQ7bSbyiSqNW/UOPWDmzC7cXskL1hekSITh1Oc6JV0XCWWZ9DE8VYSB71xocLk3gmGw==", - "dev": true - }, "is-absolute": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-absolute/-/is-absolute-1.0.0.tgz", @@ -4140,7 +4105,7 @@ }, "is-obj": { "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", + "resolved": "http://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=" }, "is-path-cwd": { @@ -4192,12 +4157,6 @@ "has": "^1.0.1" } }, - "is-regexp": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-regexp/-/is-regexp-1.0.0.tgz", - "integrity": "sha1-/S2INUXEa6xaYz57mgnof6LLUGk=", - "dev": true - }, "is-relative": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-relative/-/is-relative-1.0.0.tgz", @@ -4767,11 +4726,6 @@ "p-is-promise": "^2.0.0" } }, - "memory": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/memory/-/memory-0.0.3.tgz", - "integrity": "sha1-zgCclqXIreLyz0psmmlCprX+JvU=" - }, "meow": { "version": "3.7.0", "resolved": "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz", @@ -5002,11 +4956,6 @@ "lower-case": "^1.1.1" } }, - "node-free": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-free/-/node-free-1.0.0.tgz", - "integrity": "sha1-0rgIX0fCNh7dGfX/OpdKnLpJiwc=" - }, "node-int64": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.3.3.tgz", @@ -6567,7 +6516,7 @@ }, "pause-stream": { "version": "0.0.11", - "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", + "resolved": "http://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", "integrity": "sha1-/lo0sMvOErWqaitAPuLnO2AvFEU=", "dev": true, "requires": { @@ -6684,15 +6633,6 @@ } } }, - "plur": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/plur/-/plur-3.1.1.tgz", - "integrity": "sha512-t1Ax8KUvV3FFII8ltczPn2tJdjqbd1sIzu6t4JL7nQ3EyeL/lTrj5PWKb06ic5/6XYDr65rQ4uzQEGN70/6X5w==", - "dev": true, - "requires": { - "irregular-plurals": "^2.0.0" - } - }, "pm2": { "version": "3.5.0", "resolved": "https://registry.npmjs.org/pm2/-/pm2-3.5.0.tgz", @@ -6834,7 +6774,7 @@ }, "pretty-hrtime": { "version": "1.0.3", - "resolved": "https://registry.npmjs.org/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz", + "resolved": "http://registry.npmjs.org/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz", "integrity": "sha1-t+PqQkNaTJsnWdmeDyAesZWALuE=", "dev": true }, @@ -6929,20 +6869,20 @@ "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.14.0.tgz", "integrity": "sha512-SayS2wUX/8LF8Yo2Rkpc5nkAu4Jg3qu+OLTDSOZtisVQMB2Z5vjlY2TdPi/5CgZKiZroYIiyUN3sRX63El9iaw==", "requires": { - "debug": "^4.1.0", - "extract-zip": "^1.6.6", - "https-proxy-agent": "^2.2.1", + "debug": "^3.1.0", + "extract-zip": "^1.6.5", + "https-proxy-agent": "^2.1.0", "mime": "^2.0.3", - "progress": "^2.0.1", + "progress": "^2.0.0", "proxy-from-env": "^1.0.0", "rimraf": "^2.6.1", - "ws": "^6.1.0" + "ws": "^3.0.0" }, "dependencies": { "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", "requires": { "ms": "^2.1.1" } @@ -6957,7 +6897,9 @@ "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz", "integrity": "sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA==", "requires": { - "async-limiter": "~1.0.0" + "async-limiter": "~1.0.0", + "safe-buffer": "~5.1.0", + "ultron": "~1.1.0" } } } @@ -7669,7 +7611,7 @@ "dependencies": { "ansi-regex": { "version": "0.2.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-0.2.1.tgz", + "resolved": "http://registry.npmjs.org/ansi-regex/-/ansi-regex-0.2.1.tgz", "integrity": "sha1-DY6UaWej2BQ/k+JOKYUl/BsiNfk=", "dev": true }, @@ -7681,7 +7623,7 @@ }, "chalk": { "version": "0.5.1", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-0.5.1.tgz", + "resolved": "http://registry.npmjs.org/chalk/-/chalk-0.5.1.tgz", "integrity": "sha1-Zjs6ZItotV0EaQ1JFnqoN4WPIXQ=", "dev": true, "requires": { @@ -7709,7 +7651,7 @@ }, "strip-ansi": { "version": "0.3.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-0.3.0.tgz", + "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-0.3.0.tgz", "integrity": "sha1-JfSOoiynkYfzF0pNuHWTR7sSYiA=", "dev": true, "requires": { @@ -7731,7 +7673,7 @@ }, "stream-combiner": { "version": "0.2.2", - "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz", + "resolved": "http://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz", "integrity": "sha1-rsjLrBd7Vrb0+kec7YwZEs7lKFg=", "dev": true, "requires": { @@ -7813,17 +7755,6 @@ "safe-buffer": "~5.1.0" } }, - "stringify-object": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/stringify-object/-/stringify-object-3.3.0.tgz", - "integrity": "sha512-rHqiFh1elqCQ9WPLIC8I0Q/g/wj5J1eMkyoiD6eoQApWHP0FtlK7rqnhmabL5VUY9JQCcqwwvlOaSuutekgyrw==", - "dev": true, - "requires": { - "get-own-enumerable-property-symbols": "^3.0.0", - "is-obj": "^1.0.1", - "is-regexp": "^1.0.0" - } - }, "strip-ansi": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", @@ -8579,7 +8510,7 @@ }, "readable-stream": { "version": "1.0.34", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", + "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz", "integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=", "dev": true, "requires": { diff --git a/package.json b/package.json index cb19b6e..0282f6d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "obdfcascrape", "version": "1.0.0", - "description": "", + "description": "Scraping system to extract data from European Bank and Financial Services", "main": "index.js", "scripts": { "test": "nyc tape tests/**/*.js", @@ -15,7 +15,7 @@ "malta": "node mt.js", "debuglogs": "node debuglogs.js" }, - "author": "", + "author": "Martin Donnelly", "license": "ISC", "dependencies": { "archiver": "^2.1.1", @@ -39,7 +39,7 @@ "moment": "^2.24.0", "node-free": "^1.0.0", "pm2": "^3.5.0", - "puppeteer": "^1.14.0", + "puppeteer": "1.15.0", "remove-accents-diacritics": "^1.0.2", "request": "^2.88.0", "tld-extract": "^1.0.1", diff --git a/tests/data/at/ent_001.html b/tests/data/at/ent_001.html new file mode 100644 index 0000000..127f619 --- /dev/null +++ b/tests/data/at/ent_001.html @@ -0,0 +1,22 @@ +

+ Intermarket Bank AG Wien + +

+ +
+
    + +
  • Category: Banks - Banks licensed in Austria
    Banks - Austrian Banks in the EEA (freedom to provide services)
  • Address: Am Belvedere 1 | 1100 Wien | Austria
  • Contact:
  • Legal Identifiers:
    • Bank identification number: 73600
    • Commercial register number: 94144y
    • +
+ +

Show licenses

+ +
+ + Print view (Opens in new tab) +
+ +
diff --git a/tests/data/at/ent_001.json b/tests/data/at/ent_001.json new file mode 100644 index 0000000..0de2902 --- /dev/null +++ b/tests/data/at/ent_001.json @@ -0,0 +1,42 @@ +{ + "name": "Intermarket Bank AG", + "address": "Am Belvedere 1, 1100 Wien, Austria", + "phone": "+43 5010028900", + "email": "Meldewesen@Intermarket.at", + "website": "www.intermarket.at", + "bankIdentificationNumber": "73600", + "commercialRegisterNumber": "94144y", + "categories": [ + "Banks - Banks licensed in Austria", + "Banks - Austrian Banks in the EEA (freedom to provide services)" + ], + "permissions": [ + { + "heading": "§ 1 Abs. 1 Z 1 BWG", + "body": ["Die Entgegennahme fremder Gelder zur Verwaltung oder als Einlage (Einlagengeschäft); ausschließlich des Interbankenmarktes."] + }, + { + "heading": "§ 1 Abs. 1 Z 3 BWG", + "body": ["Der Abschluss von Geldkreditverträgen und die Gewährung von Gelddarlehen (Kreditgeschäft)"] + }, + { + "heading": "§ 1 Abs. 1 Z 4 BWG", + "body": ["Der Kauf von Schecks und Wechseln, insbesondere die Diskontierung von Wechseln (Diskontgeschäft)"] + }, + { + "heading": "§ 1 Abs. 1 Z 7 BWG", + "body": [ + "Der Handel auf eigene oder fremde Rechnung mit", + "a) ausländischen Zahlungsmitteln (Devisen- und Valutengeschäft);" + ] + }, + { + "heading": "§ 1 Abs. 1 Z 8 BWG", + "body": ["Die Übernahme von Bürgschaften, Garantien und sonstigen Haftungen für andere, sofern die übernommene Verpflichtung auf Geldleistungen lautet (Garantiegeschäft)"] + }, + { + "heading": "§ 1 Abs. 1 Z 16 BWG", + "body": ["Der Ankauf von Forderungen aus Warenlieferungen oder Dienstleistungen, die Übernahme des Risikos der Einbringlichkeit solcher Forderungen ¿ ausgenommen die Kreditversicherung - und im Zusammenhang damit der Einzug solcher Forderungen (Factoringgeschäft)"] + } + ] +} diff --git a/tests/data/at/ent_002.html b/tests/data/at/ent_002.html new file mode 100644 index 0000000..e0ea1af --- /dev/null +++ b/tests/data/at/ent_002.html @@ -0,0 +1,21 @@ +

+ Bitpanda Payments GmbH Wien + +

+ +
+
    + +
  • Category: Payment institutions - Payment Institutions licensed in Austria
  • Address: Jakov-Lind-Straße 2 | 1020 Wien | Austria
    • +
+ +

Show licenses

+ +
+ + Print view (Opens in new tab) +
+ +
diff --git a/tests/data/at/ent_002.json b/tests/data/at/ent_002.json new file mode 100644 index 0000000..f003b6c --- /dev/null +++ b/tests/data/at/ent_002.json @@ -0,0 +1,26 @@ +{ + "name": "Bitpanda Payments GmbH", + "address": "Jakov-Lind-Straße 2, 1020 Wien, Austria", + "phone": "", + "email": "", + "website": "", + "bankIdentificationNumber": "", + "commercialRegisterNumber": "", + "categories": [ + "Payment institutions - Payment Institutions licensed in Austria" + ], + "permissions": [ + { + "heading": "§ 1 Abs. 2 Z 3 ZaDiG 2018 - Zahlungsgeschäft", + "body": ["Ausführung von Zahlungsvorgängen einschließlich des Transfers von Geldbeträgen auf ein Zahlungskonto beim Zahlungsdienstleister des Zahlungsdienstnutzers oder bei einem anderen Zahlungsdienstleister (Zahlungsgeschäft) a) Ausführung von Lastschriften einschließlich einmaliger Lastschriften (Lastschriftgeschäft); b) Ausführung von Zahlungsvorgängen mittels einer Zahlungskarte oder eines ähnlichen Instruments (Zahlungskartengeschäft); c) Ausführung von Überweisungen einschließlich Daueraufträgen (Überweisungsgeschäft); Die Konzession ist jedoch dahingehend eingeschränkt, dass die angebotenen Zahlungsdienste lediglich von Kunden der Bitpanda GmbH (FN 423018k) in Anspruch genommen werden dürfen;"] + }, + { + "heading": "§ 1 Abs. 2 Z 6 ZaDiG 2018 - Finanztransfergeschäft", + "body": ["Dienste, bei denen ohne Einrichtung eines Zahlungskontos auf den Namen des Zahlers oder des Zahlungsempfängers ein Geldbetrag eines Zahlers nur zum Transfer eines entsprechenden Betrags an den Zahlungsempfänger oder an einen anderen, im Namen des Zahlungsempfängers handelnden Zahlungsdienstleister entgegengenommen wird oder bei denen der Geldbetrag im Namen des Zahlungsempfängers entgegengenommen und diesem verfügbar gemacht wird (Finanztransfergeschäft) Die Konzession ist jedoch dahingehend eingeschränkt, dass die angebotenen Zahlungsdienste lediglich von Kunden der Bitpanda GmbH (FN 423018k) in Anspruch genommen werden dürfen;"] + }, + { + "heading": "§ 1 Abs. 2 Z 7 ZaDiG 2018 - Zahlungsauslösedienst", + "body": ["Dienste, die auf Antrag des Zahlungsdienstnutzers einen Zahlungsauftrag in Bezug auf ein bei einem anderen Zahlungsdienstleister geführtes Zahlungskonto auslösen (Zahlungsauslösedienste) Die Konzession ist jedoch dahingehend eingeschränkt, dass die angebotenen Zahlungsdienste lediglich von Kunden der Bitpanda GmbH (FN 423018k) in Anspruch genommen werden dürfen;"] + } + ] +} diff --git a/tests/data/at/ent_003.html b/tests/data/at/ent_003.html new file mode 100644 index 0000000..148778b --- /dev/null +++ b/tests/data/at/ent_003.html @@ -0,0 +1,88 @@ +

+ DIMOCO Europe GmbH Brunn am Gebirge + +

+ +
+
    + +
  • Category: Payment institutions - Payment Institutions licensed in Austria
    Payment + institutions - Austrian Payment Institutions in EEA (freedom to provide services)
    • +
  • Address: Europaring F15/302 | 2345 Brunn am Gebirge | Austria
    • +
  • Legal Identifiers: +
      +
    • Commercial register number: 199901y
      • +
    +
    • +
+ +

Show + licenses

+ + +
+ + Print + view (Opens in new tab) +
+ +
diff --git a/tests/data/at/ent_003.json b/tests/data/at/ent_003.json new file mode 100644 index 0000000..654f92c --- /dev/null +++ b/tests/data/at/ent_003.json @@ -0,0 +1,59 @@ +{ + "name": "DIMOCO Europe GmbH", + "address": "Europaring F15/302, 2345 Brunn am Gebirge, Austria", + "phone": "", + "email": "", + "website": "", + "bankIdentificationNumber": "", + "commercialRegisterNumber": "199901y", + "categories": [ + "Payment institutions - Payment Institutions licensed in Austria", + "Payment institutions - Austrian Payment Institutions in EEA (freedom to provide services)" + ], + "permissions": [ + { + "heading": "§ 1 Abs. 2 Z 3 ZaDiG 2018 - Zahlungsgeschäft", + "body": ["Ausführung von Zahlungsvorgängen einschließlich des Transfers von Geldbeträgen auf ein Zahlungskonto beim Zahlungsdienstleister des Zahlungsdienstnutzers oder bei einem anderen Zahlungsdienstleister (Zahlungsgeschäft) a) Ausführung von Lastschriften einschließlich einmaliger Lastschriften (Lastschriftgeschäft); b) Ausführung von Zahlungsvorgängen mittels einer Zahlungskarte oder eines ähnlichen Instruments (Zahlungskartengeschäft); c) Ausführung von Überweisungen einschließlich Daueraufträgen (Überweisungsgeschäft);"] + }, + { + "heading": "§ 1 Abs. 2 Z 5 ZaDiG 2018 - Issuing oder Acquiring", + "body": ["Annahme und Abrechnung von Zahlungsvorgängen (Acquiring);"] + }, + { + "heading": "Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der Dienstleistungsfreiheit in Belgien", + "body": ["3. Zahlungsgeschäft"] + }, + { + "heading": "Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der Dienstleistungsfreiheit in Belgien", + "body": ["5. Zahlungsinstrumentegeschäft"] + }, + { + "heading": "Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der Dienstleistungsfreiheit in Belgien (Zusätze/Einschränkungen)", + "body": [ + "3. Ausführung von Zahlungsvorgängen einschließlich des Transfers von Geldbeträgen auf ein Zahlungskonto beim Zahlungsdienstleister des Nutzers oder bei einem anderen Zahlungsdienstleister:", + "a) Ausführung von Lastschriften einschließlich einmaliger Lastschriften", + "b) Ausführung von Zahlungsvorgängen mittels einer Zahlungskarte oder eines ähnli-chen Instruments", + "c) Ausführung von Überweisungen einschließlich Daueraufträgen", + "5. Annahme und Abrechnung (\"acquiring\") von Zahlungsinstrumenten (ohne die Gewäh-rung von Krediten entsprechend Artikel 18(4) der RL (EU) 2015/2366)" + ] + }, + { + "heading": "Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der Dienstleistungsfreiheit in Bulgarien", + "body": ["3. Zahlungsgeschäft"] + }, + { + "heading": "Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der Dienstleistungsfreiheit in Bulgarien", + "body": ["5. Zahlungsinstrumentegeschäft"] + }, + { + "heading": "Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der Dienstleistungsfreiheit in Bulgarien (Zusätze/Einschränkungen)", + "body": [ + "3. Ausführung von Zahlungsvorgängen einschließlich des Transfers von Geldbeträgen auf ein Zahlungskonto beim Zahlungsdienstleister des Nutzers oder bei einem anderen Zahlungsdienstleister:", + "a) Ausführung von Lastschriften einschließlich einmaliger Lastschriften", + "b) Ausführung von Zahlungsvorgängen mittels einer Zahlungskarte oder eines ähnli-chen Instruments", + "c) Ausführung von Überweisungen einschließlich Daueraufträgen", + "5. Annahme und Abrechnung (\"acquiring\") von Zahlungsinstrumenten (ohne die Gewäh-rung von Krediten entsprechend Artikel 18(4) der RL (EU) 2015/2366)" + ] + } + ] +} diff --git a/tests/data/be/ci_001_fullDetails.json b/tests/data/be/ci_001_fullDetails.json new file mode 100644 index 0000000..2b400a7 --- /dev/null +++ b/tests/data/be/ci_001_fullDetails.json @@ -0,0 +1,11 @@ +{ + "name": "ABN AMRO Private Banking Belgium", + "companyType": "Société anonyme", + "addressOne": "Kortrijksesteenweg 302", + "addressTwo": "9000 Gent", + "addressThree": null, + "uniqueId": "0415.835.337", + "dateOfListing": null, + "docLink": null, + "normalisedDocLink": null +} diff --git a/tests/data/be/ci_001_mainDetails.json b/tests/data/be/ci_001_mainDetails.json new file mode 100644 index 0000000..2b400a7 --- /dev/null +++ b/tests/data/be/ci_001_mainDetails.json @@ -0,0 +1,11 @@ +{ + "name": "ABN AMRO Private Banking Belgium", + "companyType": "Société anonyme", + "addressOne": "Kortrijksesteenweg 302", + "addressTwo": "9000 Gent", + "addressThree": null, + "uniqueId": "0415.835.337", + "dateOfListing": null, + "docLink": null, + "normalisedDocLink": null +} diff --git a/tests/data/be/ci_fullpage_001.html b/tests/data/be/ci_fullpage_001.html new file mode 100644 index 0000000..d1ecfee --- /dev/null +++ b/tests/data/be/ci_fullpage_001.html @@ -0,0 +1,1004 @@ + + + + + + + + + + + + + + + + + + + + + + + + + Etablissements de crédit agréés en Belgique | nbb.be + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + + +
+
+ +
+ + +

+ Etablissements de crédit agréés en Belgique

+
+ +
+ +
+
+ + + +
+
+ + + + +

Etablissements de crédit agréés en Belgique

+ + + + + +
+
+ + +
+ + +
+
+

Etablissements de crédit agréés en Belgique

+
+
+

+ Contenu +

+ + +
+
+ Articles 14 et 333, § 1er, 2° de la loi du 25 avril 2014 relative au statut et au contrôle des établissements de crédit et des sociétés de bourse +
+

+ Situation au 17-04-2019 +

+ Modifications de la liste au cours des douze derniers mois +
    +
  • +

    + 1. Etablissements de crédit de droit belge +

    +
      +
    • +

      + Banques +

      +
        +
      • + ABN AMRO Private Banking Belgium
        + Société anonyme
        + Kortrijksesteenweg 302
        + 9000 Gent
        +
        + Numéro d'identification unique :  0415.835.337 +
        • +
      • + AXA Bank Belgium
        + Société anonyme
        + Place du Trône 1
        + 1000 Bruxelles
        +
        + Numéro d'identification unique :  0404.476.835 +
        • +
      • + Banca Monte Paschi Belgio
        + Société anonyme
        + rue de la Loi 34
        + 1040 Brussel
        +
        + Numéro d'identification unique :  0403.199.306 +
        • +
      • + Bank J. Van Breda en C°
        + Société anonyme
        + Ledeganckkaai 7
        + 2000 Antwerpen
        +
        + Numéro d'identification unique :  0404.055.577 +
        • +
      • + Banque Degroof Petercam
        + Société anonyme
        + Rue de l'Industrie 44
        + 1040 Bruxelles
        +
        + Numéro d'identification unique :  0403.212.172 +
        • +
      • + Banque Eni
        + Société anonyme
        + Rue Guimard 1A
        + 1040 Bruxelles
        +
        + Numéro d'identification unique :  0879.995.183 +
        • +
      • + Banque Nagelmackers
        + Société anonyme
        + Avenue de l'Astronomie 23
        + 1210 Bruxelles
        +
        + Numéro d'identification unique :  0404.140.107 +
        • +
      • + Banque Transatlantique Belgium, en abrégé : BTB
        + Société anonyme
        + Rue De Crayer 14
        + 1000 Bruxelles
        +
        + Numéro d'identification unique :  0872.743.345 +
        • +
      • + Belfius Banque
        + Société anonyme
        + Place Charles Rogier 11
        + 1210 Bruxelles
        +
        + Numéro d'identification unique :  0403.201.185 +
        • +
      • + Beobank SA
        + Société anonyme
        + Boulevard Général Jacques 263g
        + 1050 Bruxelles
        +
        + Numéro d'identification unique :  0401.517.147 +
        • +
      • + BNP Paribas Fortis
        + Société anonyme
        + Montagne du Parc 3
        + 1000 Bruxelles
        +
        + Numéro d'identification unique :  0403.199.702 +
        • +
      • + bpost banque
        + Société anonyme
        + rue du Marquis 1
        + 1000 Bruxelles
        +
        + Numéro d'identification unique :  0456.038.471 +
        • +
      • + Byblos Bank Europe
        + Société anonyme
        + Rue Montoyer 10 bte 3
        + 1000 Bruxelles
        +
        + Numéro d'identification unique :  0415.718.640 +
        • +
      • + CBC Banque
        + Société anonyme
        + Avenue Albert 1er 60
        + 5000 Namur
        +
        + Numéro d'identification unique :  0403.211.380 +
        • +
      • + Delen Private Bank
        + Société anonyme
        + Jan Van Rijswijcklaan 184
        + 2020 Antwerpen
        +
        + Numéro d'identification unique :  0453.076.211 +
        • +
      • + Euroclear Bank
        + Société anonyme
        + Boulevard du Roi Albert II 1
        + 1210 Bruxelles
        +
        + Numéro d'identification unique :  0429.875.591 +
        • +
      • + Europabank
        + Société anonyme
        + Burgstraat 170
        + 9000 Gent
        +
        + Numéro d'identification unique :  0400.028.394 +
        • +
      • + ING Belgique
        + Société anonyme
        + Avenue Marnix 24
        + 1000 Bruxelles
        +
        + Numéro d'identification unique :  0403.200.393 +
        • +
      • + KBC Bank
        + Société anonyme
        + Avenue du Port 2
        + 1080 Bruxelles
        +
        + Numéro d'identification unique :  0462.920.226 +
        • +
      • + MeDirect Bank SA
        + Société anonyme
        + boulevard de l'Impératrice 66
        + 1000 Bruxelles
        +
        + Numéro d'identification unique :  0553.851.093 +
        • +
      • + Puilaetco Dewaay Private Bankers
        + Société anonyme
        + Avenue Herrmann Debroux 46
        + 1160 Bruxelles
        +
        + Numéro d'identification unique :  0403.236.126 +
        • +
      • + Santander Consumer Bank
        + Société anonyme
        + Avenue des Nerviens 85
        + 1040 Bruxelles
        +
        + Numéro d'identification unique :  0464.657.318 +
        • +
      • + Shizuoka Bank (Europe)
        + Société anonyme
        + Rue Jules Cockx 8-10 bte 9
        + 1160 Bruxelles
        +
        + Numéro d'identification unique :  0442.744.424 +
        • +
      • + The Bank of New York Mellon SA
        + Société anonyme
        + Rue Montoyer 46
        + 1000 Bruxelles
        +
        + Numéro d'identification unique :  0806.743.159 +
        • +
      • + United Taiwan Bank
        + Société anonyme
        + Square de Meeûs 1
        + 1er étage
        + 1000 Bruxelles
        +
        + Numéro d'identification unique :  0448.490.386 +
        • + +
      +
      • +
    • +

      + Banques d'épargne ou caisses d'épargne +

      +
        +
      • + Argenta Banque d'Epargne, en abrégé : ASPA
        + Société anonyme
        + Belgiëlei 49-53
        + 2018 Antwerpen
        +
        + Numéro d'identification unique :  0404.453.574 +
        • +
      • + Banque CPH, en abrégé : CPH
        + Société coopérative
        + Rue Perdue 7
        + 7500 Tournai
        +
        + Numéro d'identification unique :  0402.487.939 +
        • +
      • + Centrale Kredietverlening, en abrégé : CKV
        + Société anonyme
        + Mannebeekstraat 33
        + 8790 Waregem
        +
        + Numéro d'identification unique :  0400.040.965 +
        • +
      • + vdk bank
        + Société anonyme
        + Sint-Michielsplein 16
        + 9000 Gent
        +
        + Numéro d'identification unique :  0400.067.788 +
        • + +
      +
        +
      • +
        + Fédérations d'établissements de crédit +
        +
          +
        • + Crelan
          + Société anonyme
          + Boulevard Sylvain Dupuis 251
          + 1070 Bruxelles
          +
          + Numéro d'identification unique :  0205.764.318 +
          • + +
        +
        • + +
      +
      • +
    • +

      + Banques de titres +

      +
        +
      • + Bank Dierickx Leys
        + Société anonyme
        + Kasteelpleinstraat 44
        + 2000 Antwerpen
        +
        + Numéro d'identification unique :  0413.564.943 +
        • +
      • + van de Put & Co Banquiers Privés
        + Société en commandite par actions
        + Van Putlei 74-76
        + 2018 Antwerpen
        +
        + Numéro d'identification unique :  0404.501.381 +
        • + +
      +
      • + +
    +
    • +
  • +

    + 2. Succursales en Belgique des établissements de crédit relevant du droit d'un Etat non membre de l'Espace économique européen +

    +
      +
    • + Etats-Unis d'Amérique +
        +
      • + JPMorgan Chase Bank, National Association
        + Boulevard du Roi Albert II 1
        + 1210 Bruxelles
        +
        + Numéro d'identification unique :  0427.527.696 +
        • +
      • + The Bank of New York Mellon (Brussels Branch)
        + Rue Montoyer 46
        + 1000 Bruxelles
        +
        + Numéro d'identification unique :  0456.333.827 +
        • + +
      + +
      • +
    • + Inde +
        +
      • + Bank of Baroda
        + Rue de la loi 28
        + 1040 Bruxelles
        +
        + Numéro d'identification unique :  0416.144.648 +
        • +
      • + Bank of India
        + Schupstraat 18-20
        + 2018 Antwerpen
        +
        + Numéro d'identification unique :  0889.389.238 +
        • +
      • + State Bank of India
        + Korte Herentalsestraat 3
        + 2018 Antwerpen
        +
        + Numéro d'identification unique :  0424.025.008 +
        • + +
      + +
      • +
    • + Japon +
        +
      • + Sumitomo Mitsui Banking Corporation
        + Rue Montoyer 51 bte 6
        + 1000 Bruxelles
        +
        + Numéro d'identification unique :  0413.172.884 +
        • + +
      + +
      • +
    • + Pakistan +
        +
      • + Habib Bank Ltd
        + rue du Trône 14/16
        + 1000 Brussel
        +
        + Numéro d'identification unique :  0415.466.440 +
        • + +
      + +
      • + +
    +
    • +
  • +

    + Annexe 1 : Etablissements de crédit affiliés à un organisme central avec lequel ils forment une fédération +

    +
      +
    • + CrelanCo
      + Société coopérative à responsabilité limitée
      + Boulevard Sylvain Dupuis 251
      + 1070 Bruxelles
      +
      + Numéro d'identification unique :  0403.263.840 +
      • + +
    +
    • +
  • +

    + Annexe 2 : Compagnies financières de droit belge +

    +
      +
    • + Crédit Mutuel Nord Europe Belgium, en abrégé : CMNE Belgium
      + Société anonyme
      + Boulevard de Waterloo 16
      + 1000 Bruxelles
      +
      + Numéro d'identification unique :  0470.114.260 +
      • +
    • + Datex
      + Société anonyme
      + Mannebeekstraat 33
      + 8790 Waregem
      +
      + Numéro d'identification unique :  0416.296.185 +
      • +
    • + Dexia
      + Société anonyme
      + Place du Champ de Mars 5
      + 1050 Bruxelles
      +
      + Numéro d'identification unique :  0458.548.296 +
      • +
    • + Euroclear Holding
      + Société anonyme
      + Boulevard du Roi Albert II 1
      + 1210 Bruxelles
      +
      + Numéro d'identification unique :  0700.808.073 +
      • +
    • + Euroclear, en abrégé : ESA, SA
      + Société anonyme
      + Boulevard du Roi Albert II 1
      + 1210 Bruxelles
      +
      + Numéro d'identification unique :  0423.747.369 +
      • +
    • + FinAx
      + Société anonyme
      + Begijnenvest 113
      + 2000 Antwerpen
      +
      + Numéro d'identification unique :  0718.694.279 +
      • + +
    +
    • +
  • +

    + Annexe 3 : Etablissements financiers de droit belge, filiales d'un ou plusieurs établissements de crédit de droit belge, qui sont autorisés à exercer leurs activités dans d'autres états membres de l'Espace Economique Européen +

    +
      +
    • +

      + Via la libre prestation de services +

      +
        +
      • + Alpha Credit
        + Société anonyme
        + Rue Ravenstein 60 bte 15
        + 1000 Bruxelles
        +
        + Numéro d'identification unique :  0445.781.316 +
        • +
      • + Belfius Commercial Finance
        + Société anonyme
        + Place Rogier 11
        + 1210 Bruxelles
        +
        + Numéro d'identification unique :  0440.627.349 +
        • +
      • + BNP Paribas Fortis Factor
        + Société anonyme
        + Steenweg op Tielen 51
        + 2300 Turnhout +
        • +
      • + ING Commercial Finance Belux
        + Société anonyme
        + Cours Saint-Michel 60
        + 1040 Bruxells
        +
        + Numéro d'identification unique :  0470.131.086 +
        • +
      • + ING Lease Belgium
        + Société anonyme
        + Cours Saint-Michel 60
        + 1040 Bruxelles
        +
        + Numéro d'identification unique :  0402.918.402 +
        • +
      • + KBC Commercial Finance
        + Société anonyme
        + Avenue du Port 6
        + 1080 Bruxelles
        +
        + Numéro d'identification unique :  0403.278.488 +
        • + +
      +
      • + +
    +
    • + +
+

+ Modifications de la liste au cours des douze derniers mois +

+
+
    +
  • +

    + 1. Etablissements de crédit de droit belge +

    +
      +
    • + Radiation +
      + + + + + + + + + + + + + + +
      Date effective
      30-04-2018Record Bank
      +
      +
      • +
    • + Changement de la dénomination sociale +
      + + + + + + + + + + + + + + + + +
      Date effectiveAncienne dénomination socialeNouvelle dénomination sociale
      28-02-2019Société Générale Private Banking ABN AMRO Private Banking Belgium
      +
      +
      • +
    • + Changement de l'adresse +
      + + + + + + + + + + + + + + + + + + +
      Date effective Ancienne adresseNouvelle adresse
      28-05-2018Belfius Banque Boulevard Pachéco 44
      1000 Bruxelles
      Belgique       		Place Charles Rogier 11
      1210 Bruxelles
      Belgique
      +
      +
      • + +
    + +
    • +
  • +

    + 2. Succursales en Belgique des établissements de crédit relevant du droit d'un Etat non membre de l'Espace économique européen +

    +
      +
    • + Radiation +
      + + + + + + + + + + + + + + +
      Date effective
      26-03-2019Union Bank of India
      +
      +
      • + +
    + +
    • +
  • +

    + Annexe 2 : Compagnies financières de droit belge +

    +
      +
    • + Agrément +
      + + + + + + + + + + + + + + + + + + + +
      Date effective
      13-11-2018Euroclear Holding
      31-12-2018FinAx
      +
      +
      • +
    • + Radiation +
      + + + + + + + + + + + + + + +
      Date effective
      31-12-2018Finaxis
      +
      +
      • + +
    + +
    • + +
+ +
+
+ + +
+

Fin de liste

+ Haut de liste +
+
+ + +
+
+
+
+ + +
+ + +
+
+ + + + + +
+
+ + + + diff --git a/tests/data/be/em_001_fullDetails.json b/tests/data/be/em_001_fullDetails.json new file mode 100644 index 0000000..909d698 --- /dev/null +++ b/tests/data/be/em_001_fullDetails.json @@ -0,0 +1,14 @@ +{ + "name": "Buy Way Personal Finance", + "companyType": "Société anonyme", + "addressOne": "Rue de l'Evêque 26", + "addressTwo": "1000 Bruxelles", + "addressThree": null, + "uniqueId": "0400.282.277", + "dateOfListing": "07-05-2013", + "docLink": "http://www.nbb.be/DOC/CP/ENG/psd/Buy%20Way.docx?t=19418", + "normalisedDocLink": "http://www.nbb.be/DOC/CP/ENG/psd/Buy%20Way.docx?t=19418", + "activitesGenerales": [ "A", "B" ], + "servicesDePaiement": [ "1", "2", "3", "4", "5", "6", "7", "8" ], + "autresServicesLies": [ "X" ] +} diff --git a/tests/data/be/em_001_mainDetails.json b/tests/data/be/em_001_mainDetails.json new file mode 100644 index 0000000..f059a6a --- /dev/null +++ b/tests/data/be/em_001_mainDetails.json @@ -0,0 +1,11 @@ +{ + "name": "Ingenico Financial Solutions", + "companyType": "Société anonyme", + "addressOne": "Leonardo Da Vincilaan 3", + "addressTwo": "Corporate Village, Bayreuth Building", + "addressThree": "1930 Zaventem", + "uniqueId": "0886.476.763", + "dateOfListing": "30-11-2012", + "docLink": "http://www.nbb.be/DOC/CP/ENG/psd/Ingenico.docx?t=19418", + "normalisedDocLink": "http://www.nbb.be/DOC/CP/ENG/psd/Ingenico.docx?t=19418" +} diff --git a/tests/data/be/em_fullpage_001.html b/tests/data/be/em_fullpage_001.html new file mode 100644 index 0000000..c1138eb --- /dev/null +++ b/tests/data/be/em_fullpage_001.html @@ -0,0 +1,509 @@ + + + + + + + + + + + + + + + + + + + + + + + + + Etablissements de monnaie électronique agréés en Belgique | nbb.be + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + + +
+
+ +
+ + +

+ Etablissements de monnaie électronique agréés en Belgique

+
+ +
+ +
+
+ + + +
+
+ + + + +

Etablissements de monnaie électronique agréés en Belgique

+ + + + + +
+
+ + +
+ + +
+
+

Etablissements de monnaie électronique agréés en Belgique

+
+
+

+ Contenu +

+ + +
+
+ Article 166 §1, 1° de la loi du 11 mars 2018 relative au statut et au contrôle des établissements de paiement et des établissements de monnaie électronique, à l'accès à l'activité de prestataire de services de paiement, et à l'activité d'émission de monnaie électronique, et à l'accès aux systèmes de paiement +
+
+

(*) Activités générales

+
    +
  1. Emission de monnaie électronique
    2. +
  2. Distribution/remboursement de monnaie électronique
    3. +
+

(**) Services de paiement

+
    +
  1. Les services permettant de verser des espèces sur un compte de paiement et toutes les opérations qu’exige la gestion d’un compte de paiement;
    2. +
  2. Les services permettant de retirer des espèces d’un compte de paiement et toutes les opérations qu’exige la gestion d’un compte de paiement;
    3. +
  3. L’exécution d’opérations de paiement, y compris les transferts de fonds sur un compte de paiement auprès du prestataire de services de paiement de l’utilisateur ou auprès d’un autre prestataire de services de paiement : +
      +
    1. l’exécution de prélèvements, y compris de prélèvements autorisés unitairement,
      2. +
    2. l’exécution d’opérations de paiement à l’aide d’une carte de paiement ou d’un dispositif similaire,
      3. +
    3. l’exécution de virements, y compris d’ordres permanents;
      4. +
    +
    4. +
  4. L’exécution d’opérations de paiement dans le cadre desquelles les fonds sont couverts par une ligne de crédit accordée à l’utilisateur de services de paiement : +
      +
    1. l’exécution de prélèvements, y compris de prélèvements autorisés unitairement,
      2. +
    2. l’exécution d’opérations de paiement à l’aide d’une carte de paiement ou d’un dispositif similaire,
      3. +
    3. l’exécution de virements, y compris d’ordres permanents;
      4. +
    +
    5. +
  5. L’émission d’instruments de paiement et/ou l’acquisition d’opérations de paiement;
    6. +
  6. Les transmissions de fonds;
    7. +
  7. Les services d’initiation de paiement;
    8. +
  8. Les services d’information sur les comptes;
    9. +
+

(***) Autres services liés

+
    +
  1. Octroi de crédits liés aux services de paiement visés aux points 4, 5 ou 7 des services de paiement visés sous (*)
    2. +
+ +
+

+ Situation au 08-04-2019 +

+ Modifications de la liste au cours des douze derniers mois +
    +
  • +

    + Etablissements de monnaie électronique de droit belge +

    +
    + Nombre total d'établissements :  6 +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Activités générales
    (*)
    		Services de paiement
    (**)
    		Autres services liés
    (***)
    Buy Way Personal Finance
    + Société anonyme
    + Rue de l'Evêque 26
    + 1000 Bruxelles
    +
    + Numéro d'identification unique :  0400.282.277
    + Date de l'inscription à la liste :  07-05-2013
    + Succursales/Agents/Distributeurs/Libre prestation des services :  voyez la liste en annexe    		A B1 2 3 4 5 6 7 8X
    Fimaser
    + Société anonyme
    + Avenue des Olympiades 20
    + 1140 Bruxelles
    +
    + Numéro d'identification unique :  0434.818.930
    + Date de l'inscription à la liste :  17-12-2013
    + Succursales/Agents/Distributeurs/Libre prestation des services :  voyez la liste en annexe    		A B1 2 3 4 5 6X
    HiPay ME
    + Société anonyme
    + Avenue des Volontaires 19
    + 1160 Bruxelles
    +
    + Numéro d'identification unique :  0897.928.802
    + Date de l'inscription à la liste :  30-11-2012
    + Succursales/Agents/Distributeurs/Libre prestation des services :  voyez la liste en annexe    		A B3
    Imagor
    + Société anonyme
    + Boulevard de la Plaine 15
    + 1050 Bruxelles
    +
    + Numéro d'identification unique :  0461.328.436
    + Date de l'inscription à la liste :  18-06-2013
    + Succursales/Agents/Distributeurs/Libre prestation des services :  voyez la liste en annexe    		A B1 2 3 5
    Ingenico Financial Solutions
    + Société anonyme
    + Leonardo Da Vincilaan 3
    + Corporate Village Bayreuth Building
    + 1930 Zaventem
    +
    + Numéro d'identification unique :  0886.476.763
    + Date de l'inscription à la liste :  30-11-2012
    + Succursales/Agents/Distributeurs/Libre prestation des services :  voyez la liste en annexe    		A B3 4 5
    Paynovate
    + Société anonyme
    + Cantersteen 47
    + 1000 Bruxelles
    +
    + Numéro d'identification unique :  0506.763.929
    + Date de l'inscription à la liste :  26-03-2019    		A3 4 5
    + +
    +
    • + +
+

+ Modifications de la liste au cours des douze derniers mois +

+
+
    +
  • +

    + Etablissements de monnaie électronique de droit belge +

    +
      +
    • + Agrément +
      + + + + + + + + + + + + + + +
      Date effective
      26-03-2019Paynovate
      +
      +
      • +
    • + Changement de la dénomination sociale +
      + + + + + + + + + + + + + + + + +
      Date effectiveAncienne dénominationNouvelle dénomination
      17-05-2018Hi-Media Porte Monnaie Electronique (en abrégré HPME) HiPay ME
      +
      +
      • +
    • + Changement de l'adresse +
      + + + + + + + + + + + + + + + + + + +
      Date effective Ancienne adresseNouvelle adresse
      04-03-2019Ingenico Financial Solutions Boulevard de la Woluwe 102
      1200 Bruxelles
      Belgique       		Leonardo Da Vincilaan 3
      Corporate Village Bayreuth Building
      1930 Zaventem
      Belgique
      +
      +
      • + +
    + +
    • + +
+ +
+

Fin de liste

+ Haut de liste +
+
+ + +
+
+
+
+ + +
+ + +
+
+ + + + + +
+
+ + + + diff --git a/tests/data/be/ps_001.html b/tests/data/be/ps_001.html new file mode 100644 index 0000000..83d5abb --- /dev/null +++ b/tests/data/be/ps_001.html @@ -0,0 +1,10 @@ +Airplus International
+ Société anonyme
+ Boulevard de l'Impératrice 66
+ 1000 Bruxelles
+
+ Numéro d'identification unique :  0883.523.807
+ Date de l'inscription à la liste :  26-04-2011
+ Succursales/Agents/Libre prestation des services :  voyez la liste en annexe +2 3 4 5 +X diff --git a/tests/data/be/ps_001.json b/tests/data/be/ps_001.json new file mode 100644 index 0000000..d59f646 --- /dev/null +++ b/tests/data/be/ps_001.json @@ -0,0 +1,13 @@ +{ + "name": "Airplus International", + "companyType": "Société anonyme", + "addressOne": "Boulevard de l'Impératrice 66", + "addressTwo": "1000 Bruxelles", + "uniqueId": "0883.523.807", + "dateOfListing": "26-04-2011", + "paymentServices": [ + "2", "3", "4", "5" + ], + "otherRelatedServices": "X", + "docLink": "http://www.nbb.be/DOC/CP/ENG/psd/BCCC.docx?t=1949" +} diff --git a/tests/data/be/ps_001_fullDetails.json b/tests/data/be/ps_001_fullDetails.json new file mode 100644 index 0000000..2a99781 --- /dev/null +++ b/tests/data/be/ps_001_fullDetails.json @@ -0,0 +1,13 @@ +{ + "name": "Airplus International", + "companyType": "Société anonyme", + "addressOne": "Boulevard de l'Impératrice 66", + "addressTwo": "1000 Bruxelles", + "addressThree": null, + "uniqueId": "0883.523.807", + "dateOfListing": "26-04-2011", + "docLink": "http://www.nbb.be/DOC/CP/ENG/psd/BCCC.docx?t=19417", + "normalisedDocLink": "http://www.nbb.be/DOC/CP/ENG/psd/BCCC.docx?t=19417", + "servicesDePaiement": [ "2", "3", "4", "5" ], + "autresServicesLies": [ "X" ] +} diff --git a/tests/data/be/ps_001_mainDetails.json b/tests/data/be/ps_001_mainDetails.json new file mode 100644 index 0000000..13f623e --- /dev/null +++ b/tests/data/be/ps_001_mainDetails.json @@ -0,0 +1,11 @@ +{ + "name": "Airplus International", + "companyType": "Société anonyme", + "addressOne": "Boulevard de l'Impératrice 66", + "addressTwo": "1000 Bruxelles", + "addressThree": null, + "uniqueId": "0883.523.807", + "dateOfListing": "26-04-2011", + "docLink": "http://www.nbb.be/DOC/CP/ENG/psd/BCCC.docx?t=19417", + "normalisedDocLink": "http://www.nbb.be/DOC/CP/ENG/psd/BCCC.docx?t=19417" +} diff --git a/tests/data/be/ps_fullpage_001.html b/tests/data/be/ps_fullpage_001.html new file mode 100644 index 0000000..fa76f65 --- /dev/null +++ b/tests/data/be/ps_fullpage_001.html @@ -0,0 +1,751 @@ + + + + + + + + + + + + + + + + + + + + + + + + + Etablissements de paiement agréés en Belgique | nbb.be + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + + +
+
+ +
+ + +

+ Etablissements de paiement agréés en Belgique

+
+ +
+ +
+
+ + + +
+
+ + + + +

Etablissements de paiement agréés en Belgique

+ + + + + +
+
+ + +
+ + +
+
+

Etablissements de paiement agréés en Belgique

+
+
+

+ Contenu +

+ + +
+
+ Article 8 §1, 1° de la loi du 11 mars 2018 relative au statut et au contrôle des établissements de paiement et des établissements de monnaie électronique, à l'accès à l'activité de prestataire de services de paiement, et à l'activité d'émission de monnaie électronique, et à l'accès aux systèmes de paiement +
+
+

(*) Services de paiement

+
    +
  1. Les services permettant de verser des espèces sur un compte de paiement et toutes les opérations qu’exige la gestion d’un compte de paiement;
    2. +
  2. Les services permettant de retirer des espèces d’un compte de paiement et toutes les opérations qu’exige la gestion d’un compte de paiement;
    3. +
  3. L’exécution d’opérations de paiement, y compris les transferts de fonds sur un compte de paiement auprès du prestataire de services de paiement de l’utilisateur ou auprès d’un autre prestataire de services de paiement : +
      +
    1. l’exécution de prélèvements, y compris de prélèvements autorisés unitairement,
      2. +
    2. l’exécution d’opérations de paiement à l’aide d’une carte de paiement ou d’un dispositif similaire,
      3. +
    3. l’exécution de virements, y compris d’ordres permanents;
      4. +
    +
    4. +
  4. L’exécution d’opérations de paiement dans le cadre desquelles les fonds sont couverts par une ligne de crédit accordée à l’utilisateur de services de paiement : +
      +
    1. l’exécution de prélèvements, y compris de prélèvements autorisés unitairement,
      2. +
    2. l’exécution d’opérations de paiement à l’aide d’une carte de paiement ou d’un dispositif similaire,
      3. +
    3. l’exécution de virements, y compris d’ordres permanents;
      4. +
    +
    5. +
  5. L’émission d’instruments de paiement et/ou l’acquisition d’opérations de paiement;
    6. +
  6. Les transmissions de fonds;
    7. +
  7. Les services d’initiation de paiement;
    8. +
  8. Les services d’information sur les comptes;
    9. +
+

(**) Autres services liés

+
    +
  1. Octroi de crédits liés aux services de paiement visés aux points 4, 5 ou 7 des services de paiement visés sous (*)
    2. +
+ +
+

+ Situation au 08-04-2019 +

+ Modifications de la liste au cours des douze derniers mois +
    +
  • +

    + Etablissements de paiement de droit belge +

    +
    + Nombre total d'établissements :  24 +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Services de paiement
    (*)
    		Autres services liés
    (**)
    Airplus International
    + Société anonyme
    + Boulevard de l'Impératrice 66
    + 1000 Bruxelles
    +
    + Numéro d'identification unique :  0883.523.807
    + Date de l'inscription à la liste :  26-04-2011
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		2 3 4 5X
    Alpha Card
    + Société coopérative à responsabilité limitée
    + Boulevard du Souverain 100
    + 1170 Watermael Boitsfort
    +
    + Numéro d'identification unique :  0463.926.551
    + Date de l'inscription à la liste :  26-04-2011
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		2 3 4 5X
    Alpha Card Merchant Services
    + Société coopérative à responsabilité limitée
    + Boulevard du Souverain 100
    + 1170 Watermael Boitsfort
    +
    + Numéro d'identification unique :  0475.933.171
    + Date de l'inscription à la liste :  26-04-2011
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		2 3 4 5X
    Belmoney Transfert
    + Société privée à responsabilité limitée
    + Place Bara 28
    + 1070 Bruxelles
    +
    + Numéro d'identification unique :  0540.745.997
    + Date de l'inscription à la liste :  20-03-2018
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		6
    Cashfree
    + Société anonyme
    + Mortelputstraat 49
    + 9830 Sint-Martens-Latem
    +
    + Numéro d'identification unique :  0663.774.859
    + Date de l'inscription à la liste :  11-07-2017    		3 4 5
    Cofidis
    + Société anonyme
    + Chaussée de Lille 422A
    + 7501 Orcq
    +
    + Numéro d'identification unique :  0400.359.283
    + Date de l'inscription à la liste :  07-05-2013    		4 5 6X
    Digiteal
    + Société anonyme
    + rue Emile Francqui 6
    + 1435 Corbais
    +
    + Numéro d'identification unique :  0630.675.588
    + Date de l'inscription à la liste :  31-10-2017
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 7 8
    Ebury Partners Belgium
    + Société anonyme
    + Boulevard du Régent 37
    + 1000 Bruxelles
    +
    + Numéro d'identification unique :  0681.746.187
    + Date de l'inscription à la liste :  31-10-2017
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 4 6
    eDebex
    + Société anonyme
    + Rue Jules Cockx 8-10
    + 1160 Bruxelles
    +
    + Numéro d'identification unique :  0502.697.352
    + Date de l'inscription à la liste :  02-02-2016
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3
    EPBF
    + Société anonyme
    + Chaussée de la Hulpe 181
    + 1170 Watermael-Boitsfort
    +
    + Numéro d'identification unique :  0837.808.497
    + Date de l'inscription à la liste :  27-06-2011
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 4 5X
    Gold Commodities Forex
    + Société anonyme
    + Winderickxplein 3 bte 2
    + 1652 Beersel
    +
    + Numéro d'identification unique :  0832.602.171
    + Date de l'inscription à la liste :  27-06-2011
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 6
    HomeSend
    + Société coopérative à responsabilité limitée
    + Rue des Colonies 56
    + 1000 Bruxelles
    +
    + Numéro d'identification unique :  0549.987.921
    + Date de l'inscription à la liste :  09-02-2016
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 4
    iBanFirst
    + Société anonyme
    + Avenue Louise 350
    + 1050 Bruxelles
    +
    + Numéro d'identification unique :  0849.872.824
    + Date de l'inscription à la liste :  22-05-2013
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 4 7 8
    Isabel
    + Société anonyme
    + Boulevard de l'Impératrice 13-15
    + 1000 Bruxelles
    +
    + Numéro d'identification unique :  0455.530.509
    + Date de l'inscription à la liste :  19-02-2019
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		7 8
    Let's Didid
    + Société anonyme
    + Montagne du Parc 3
    + 1000 Bruxelles
    +
    + Numéro d'identification unique :  0450.355.261
    + Date de l'inscription à la liste :  19-02-2019    		7 8
    Money International
    + Société anonyme
    + Chaussée de Gand 100
    + 1080 Bruxelles
    +
    + Numéro d'identification unique :  0475.445.104
    + Date de l'inscription à la liste :  26-03-2019    		6
    MoneyGram International
    + Société privée à responsabilité limitée
    + rue Joseph Stevens 7
    + 1000 Bruxelles
    +
    + Numéro d'identification unique :  0671.690.653
    + Date de l'inscription à la liste :  19-12-2017
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		6
    Moneytrans Payment Services
    + Société anonyme
    + Boulevard de Waterloo 77
    + 1000 Bruxelles
    +
    + Numéro d'identification unique :  0449.356.557
    + Date de l'inscription à la liste :  26-04-2011
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 6
    Oonex
    + Société anonyme
    + Avenue Louise 367
    + 1050 Bruxelles
    +
    + Numéro d'identification unique :  0849.185.510
    + Date de l'inscription à la liste :  23-08-2016
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 5
    PAY-NXT
    + Société anonyme
    + avenue Reine Astrid 92A
    + 1310 La Hulpe
    +
    + Numéro d'identification unique :  0649.860.804
    + Date de l'inscription à la liste :  11-10-2016
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 5
    Transferwise Europe
    + Société anonyme
    + Square de Meeûs 38 bte 40
    + 1000 Bruxelles
    +
    + Numéro d'identification unique :  0713.629.988
    + Date de l'inscription à la liste :  19-03-2019
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 5 6
    Travelex
    + Société anonyme
    + Steendam 108
    + 9000 Gent
    +
    + Numéro d'identification unique :  0451.299.329
    + Date de l'inscription à la liste :  26-04-2011
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		6
    Worldline
    + Société anonyme
    + Chaussée de Haecht 1442
    + Blue Star Building - FIN - D4 +
    + 1130 Bruxelles
    +
    + Numéro d'identification unique :  0418.547.872
    + Date de l'inscription à la liste :  21-09-2010
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 4 5 7 8
    WorldRemit Belgium
    + Société anonyme
    + Place Marcel Broodthaers 8
    + 1060 Bruxelles
    +
    + Numéro d'identification unique :  0718.634.495
    + Date de l'inscription à la liste :  26-03-2019
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		6
    + +
    +
    • + +
+

+ Modifications de la liste au cours des douze derniers mois +

+
+
    +
  • +

    + Etablissements de paiement de droit belge +

    +
      +
    • + Agrément +
      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      Date effective
      19-02-2019Isabel
      19-02-2019Let's Didid
      19-03-2019Transferwise Europe
      26-03-2019Money International
      26-03-2019WorldRemit Belgium
      +
      +
      • +
    • + Radiation +
      + + + + + + + + + + + + + + +
      Date effective
      26-06-2018B+S Payment Europe
      +
      +
      • +
    • + Changement de la dénomination sociale +
      + + + + + + + + + + + + + + + + + + + + + + +
      Date effectiveAncienne dénominationNouvelle dénomination
      04-01-2019Teal IT Digiteal
      01-02-2019Bank Card Company Corporate Airplus International
      +
      +
      • + +
    + +
    • + +
+ +
+

Fin de liste

+ Haut de liste +
+
+ + +
+
+
+
+ + +
+ + +
+
+ + + + + +
+
+ + + + diff --git a/tests/data/be/ps_index_001.html b/tests/data/be/ps_index_001.html new file mode 100644 index 0000000..ee9a4a0 --- /dev/null +++ b/tests/data/be/ps_index_001.html @@ -0,0 +1,500 @@ +
+

Etablissements de paiement agréés en Belgique

+
+
+

+ Contenu +

+ + +
+
+ Article 8 §1, 1° de la loi du 11 mars 2018 relative au statut et au contrôle des établissements de paiement et des établissements de monnaie électronique, à l'accès à l'activité de prestataire de services de paiement, et à l'activité d'émission de monnaie électronique, et à l'accès aux systèmes de paiement +
+
+

(*) Services de paiement

+
    +
  1. Les services permettant de verser des espèces sur un compte de paiement et toutes les opérations qu’exige la gestion d’un compte de paiement;
    2. +
  2. Les services permettant de retirer des espèces d’un compte de paiement et toutes les opérations qu’exige la gestion d’un compte de paiement;
    3. +
  3. L’exécution d’opérations de paiement, y compris les transferts de fonds sur un compte de paiement auprès du prestataire de services de paiement de l’utilisateur ou auprès d’un autre prestataire de services de paiement : +
      +
    1. l’exécution de prélèvements, y compris de prélèvements autorisés unitairement,
      2. +
    2. l’exécution d’opérations de paiement à l’aide d’une carte de paiement ou d’un dispositif similaire,
      3. +
    3. l’exécution de virements, y compris d’ordres permanents;
      4. +
    +
    4. +
  4. L’exécution d’opérations de paiement dans le cadre desquelles les fonds sont couverts par une ligne de crédit accordée à l’utilisateur de services de paiement : +
      +
    1. l’exécution de prélèvements, y compris de prélèvements autorisés unitairement,
      2. +
    2. l’exécution d’opérations de paiement à l’aide d’une carte de paiement ou d’un dispositif similaire,
      3. +
    3. l’exécution de virements, y compris d’ordres permanents;
      4. +
    +
    5. +
  5. L’émission d’instruments de paiement et/ou l’acquisition d’opérations de paiement;
    6. +
  6. Les transmissions de fonds;
    7. +
  7. Les services d’initiation de paiement;
    8. +
  8. Les services d’information sur les comptes;
    9. +
+

(**) Autres services liés

+
    +
  1. Octroi de crédits liés aux services de paiement visés aux points 4, 5 ou 7 des services de paiement visés sous (*)
    2. +
+ +
+

+ Situation au 08-04-2019 +

+Modifications de la liste au cours des douze derniers mois +
    +
  • +

    + Etablissements de paiement de droit belge +

    +
    + Nombre total d'établissements :  24 +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Services de paiement
    (*)
    		Autres services liés
    (**)
    Airplus International
    + Société anonyme
    + Boulevard de l'Impératrice 66
    + 1000 Bruxelles
    +
    + Numéro d'identification unique :  0883.523.807
    + Date de l'inscription à la liste :  26-04-2011
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		2 3 4 5X
    Alpha Card
    + Société coopérative à responsabilité limitée
    + Boulevard du Souverain 100
    + 1170 Watermael Boitsfort
    +
    + Numéro d'identification unique :  0463.926.551
    + Date de l'inscription à la liste :  26-04-2011
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		2 3 4 5X
    Alpha Card Merchant Services
    + Société coopérative à responsabilité limitée
    + Boulevard du Souverain 100
    + 1170 Watermael Boitsfort
    +
    + Numéro d'identification unique :  0475.933.171
    + Date de l'inscription à la liste :  26-04-2011
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		2 3 4 5X
    Belmoney Transfert
    + Société privée à responsabilité limitée
    + Place Bara 28
    + 1070 Bruxelles
    +
    + Numéro d'identification unique :  0540.745.997
    + Date de l'inscription à la liste :  20-03-2018
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		6
    Cashfree
    + Société anonyme
    + Mortelputstraat 49
    + 9830 Sint-Martens-Latem
    +
    + Numéro d'identification unique :  0663.774.859
    + Date de l'inscription à la liste :  11-07-2017    		3 4 5
    Cofidis
    + Société anonyme
    + Chaussée de Lille 422A
    + 7501 Orcq
    +
    + Numéro d'identification unique :  0400.359.283
    + Date de l'inscription à la liste :  07-05-2013    		4 5 6X
    Digiteal
    + Société anonyme
    + rue Emile Francqui 6
    + 1435 Corbais
    +
    + Numéro d'identification unique :  0630.675.588
    + Date de l'inscription à la liste :  31-10-2017
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 7 8
    Ebury Partners Belgium
    + Société anonyme
    + Boulevard du Régent 37
    + 1000 Bruxelles
    +
    + Numéro d'identification unique :  0681.746.187
    + Date de l'inscription à la liste :  31-10-2017
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 4 6
    eDebex
    + Société anonyme
    + Rue Jules Cockx 8-10
    + 1160 Bruxelles
    +
    + Numéro d'identification unique :  0502.697.352
    + Date de l'inscription à la liste :  02-02-2016
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3
    EPBF
    + Société anonyme
    + Chaussée de la Hulpe 181
    + 1170 Watermael-Boitsfort
    +
    + Numéro d'identification unique :  0837.808.497
    + Date de l'inscription à la liste :  27-06-2011
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 4 5X
    Gold Commodities Forex
    + Société anonyme
    + Winderickxplein 3 bte 2
    + 1652 Beersel
    +
    + Numéro d'identification unique :  0832.602.171
    + Date de l'inscription à la liste :  27-06-2011
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 6
    HomeSend
    + Société coopérative à responsabilité limitée
    + Rue des Colonies 56
    + 1000 Bruxelles
    +
    + Numéro d'identification unique :  0549.987.921
    + Date de l'inscription à la liste :  09-02-2016
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 4
    iBanFirst
    + Société anonyme
    + Avenue Louise 350
    + 1050 Bruxelles
    +
    + Numéro d'identification unique :  0849.872.824
    + Date de l'inscription à la liste :  22-05-2013
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 4 7 8
    Isabel
    + Société anonyme
    + Boulevard de l'Impératrice 13-15
    + 1000 Bruxelles
    +
    + Numéro d'identification unique :  0455.530.509
    + Date de l'inscription à la liste :  19-02-2019
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		7 8
    Let's Didid
    + Société anonyme
    + Montagne du Parc 3
    + 1000 Bruxelles
    +
    + Numéro d'identification unique :  0450.355.261
    + Date de l'inscription à la liste :  19-02-2019    		7 8
    Money International
    + Société anonyme
    + Chaussée de Gand 100
    + 1080 Bruxelles
    +
    + Numéro d'identification unique :  0475.445.104
    + Date de l'inscription à la liste :  26-03-2019    		6
    MoneyGram International
    + Société privée à responsabilité limitée
    + rue Joseph Stevens 7
    + 1000 Bruxelles
    +
    + Numéro d'identification unique :  0671.690.653
    + Date de l'inscription à la liste :  19-12-2017
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		6
    Moneytrans Payment Services
    + Société anonyme
    + Boulevard de Waterloo 77
    + 1000 Bruxelles
    +
    + Numéro d'identification unique :  0449.356.557
    + Date de l'inscription à la liste :  26-04-2011
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 6
    Oonex
    + Société anonyme
    + Avenue Louise 367
    + 1050 Bruxelles
    +
    + Numéro d'identification unique :  0849.185.510
    + Date de l'inscription à la liste :  23-08-2016
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 5
    PAY-NXT
    + Société anonyme
    + avenue Reine Astrid 92A
    + 1310 La Hulpe
    +
    + Numéro d'identification unique :  0649.860.804
    + Date de l'inscription à la liste :  11-10-2016
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 5
    Transferwise Europe
    + Société anonyme
    + Square de Meeûs 38 bte 40
    + 1000 Bruxelles
    +
    + Numéro d'identification unique :  0713.629.988
    + Date de l'inscription à la liste :  19-03-2019
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 5 6
    Travelex
    + Société anonyme
    + Steendam 108
    + 9000 Gent
    +
    + Numéro d'identification unique :  0451.299.329
    + Date de l'inscription à la liste :  26-04-2011
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		6
    Worldline
    + Société anonyme
    + Chaussée de Haecht 1442
    + Blue Star Building - FIN - D4 +
    + 1130 Bruxelles
    +
    + Numéro d'identification unique :  0418.547.872
    + Date de l'inscription à la liste :  21-09-2010
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		3 4 5 7 8
    WorldRemit Belgium
    + Société anonyme
    + Place Marcel Broodthaers 8
    + 1060 Bruxelles
    +
    + Numéro d'identification unique :  0718.634.495
    + Date de l'inscription à la liste :  26-03-2019
    + Succursales/Agents/Libre prestation des services :  voyez la liste en annexe    		6
    + +
    +
    • + +
+

+ Modifications de la liste au cours des douze derniers mois +

+
+
    +
  • +

    + Etablissements de paiement de droit belge +

    +
      +
    • + Agrément +
      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      Date effective
      19-02-2019Isabel
      19-02-2019Let's Didid
      19-03-2019Transferwise Europe
      26-03-2019Money International
      26-03-2019WorldRemit Belgium
      +
      +
      • +
    • + Radiation +
      + + + + + + + + + + + + + + +
      Date effective
      26-06-2018B+S Payment Europe
      +
      +
      • +
    • + Changement de la dénomination sociale +
      + + + + + + + + + + + + + + + + + + + + + + +
      Date effectiveAncienne dénominationNouvelle dénomination
      04-01-2019Teal IT Digiteal
      01-02-2019Bank Card Company Corporate Airplus International
      +
      +
      • + +
    + +
    • + +
+ +
+

Fin de liste

+Haut de liste diff --git a/tests/data/nl/din329_d2_01.html b/tests/data/nl/din329_d2_01.html new file mode 100644 index 0000000..abfc82a --- /dev/null +++ b/tests/data/nl/din329_d2_01.html @@ -0,0 +1,1324 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Information detail - De Nederlandsche Bank + + + + + + + + + + + + + + +
+ + + + + + + + +
+
+ + + + + + + + + + + + + + + + +

Information detail

+ + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ back +

+
+

+
Organization
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ 		+
+
+ Statutory name: + NIBC Bank N.V.
+ Trade name: + NIBC Direct
+ Address: + Carnegieplein 4
+
+ Postal code: + 2517 KJ
+ Place of residence: + 'S-GRAVENHAGE
+ Country: + Netherlands
+ Disclosure: +
+
+ 		+
+
+
+ 		+
+
+ Chamber of Commerce + 27032036
+ Relation number DNB: + B0685
+ Category + Emittent effecten CSDB, Bank
+
+
+
    +
  1. + Contacts +
    2. +
  2. + Application +
    3. +
  3. + EU-passport (out) +
    4. +
  4. + Covered Bonds +
    5. +
+
+
+

Contacts

+

Business address

+
+ + + + + + + + + + + +
Address:Place of residence:Country:
Carnegieplein 4'S-GRAVENHAGENetherlands
+
+

Mailing address

+
+ + + + + + + + + + + +
Address:Place of residence:Country:
Postbus 380'S-GRAVENHAGENetherlands
+
+
+
+
+
+

Application

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Financial serviceActivity / BrancheDate of entranceEnddate
2:13(1) Carrying on the business of a bank01. Acceptance of deposits and other repayable funds01-01-1988
02. Lending01-01-1988
03. Financial leasing01-01-1988
04. Payment services as referred to in Article 4(3) of Directive 2015/236601-01-1988
05. Issuing and administering means of payment (e.g. credit cards, travellers' cheques and bankers' drafts)01-01-1988
06. Guarantees and commitments01-01-1988
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);01-01-1988
7b. Foreign exchange01-01-1988
7c. Financial futures and options;01-01-1988
7d. Exchange rate and interest rate instruments01-01-1988
07e. Transferable securities01-01-1988
08. Participation in securities issues and the provision of services related to such issues01-01-1988
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakings01-01-1988
10. Money broking01-01-1988
11. Portfolio management and advice01-01-1988
12. Safekeeping and administration of securities01-01-1988
13. Credit reference services01-01-1988
14. Safe custody services01-01-1988
15. Issuance of electronic money01-01-1988
+
+
+
+
+
+

EU-passport (out)

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
EU passport (out)ActivityCountry into the EEADate of entranceEnddate
2:110 Provision of bank services to EEA01. Acceptance of deposits and other repayable fundsAustria11-04-2011
02. LendingAustria11-04-2011
06. Guarantees and commitmentsAustria11-04-2011
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);Austria11-04-2011
7b. Foreign exchangeAustria11-04-2011
7c. Financial futures and options;Austria11-04-2011
7d. Exchange rate and interest rate instrumentsAustria11-04-2011
07e. Transferable securitiesAustria11-04-2011
08. Participation in securities issues and the provision of services related to such issuesAustria11-04-2011
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsAustria11-04-2011
11. Portfolio management and adviceAustria11-04-2011
12. Safekeeping and administration of securitiesAustria11-04-2011
2:108 Branch banking to EEA01. Acceptance of deposits and other repayable fundsBelgium14-02-1996
02. LendingBelgium14-02-1996
04. Payment services as referred to in Article 4(3) of Directive 2015/2366Belgium14-02-1996
06. Guarantees and commitmentsBelgium14-02-1996
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);Belgium14-02-1996
7b. Foreign exchangeBelgium14-02-1996
7c. Financial futures and options;Belgium14-02-1996
7d. Exchange rate and interest rate instrumentsBelgium14-02-1996
07e. Transferable securitiesBelgium14-02-1996
08. Participation in securities issues and the provision of services related to such issuesBelgium14-02-1996
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsBelgium14-02-1996
10. Money brokingBelgium14-02-1996
11. Portfolio management and adviceBelgium14-02-1996
12. Safekeeping and administration of securitiesBelgium14-02-1996
13. Credit reference servicesBelgium14-02-1996
2:110 Provision of bank services to EEA01. Acceptance of deposits and other repayable fundsBelgium11-04-2011
02. LendingBelgium11-04-2011
06. Guarantees and commitmentsBelgium11-04-2011
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);Belgium11-04-2011
7b. Foreign exchangeBelgium11-04-2011
7c. Financial futures and options;Belgium11-04-2011
7d. Exchange rate and interest rate instrumentsBelgium11-04-2011
07e. Transferable securitiesBelgium11-04-2011
08. Participation in securities issues and the provision of services related to such issuesBelgium11-04-2011
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsBelgium11-04-2011
11. Portfolio management and adviceBelgium11-04-2011
12. Safekeeping and administration of securitiesBelgium11-04-2011
2:110 Provision of bank services to EEABulgaria30-06-2017
2:110 Provision of bank services to EEA01. Acceptance of deposits and other repayable fundsCyprus11-04-2011
02. LendingCyprus11-04-2011
06. Guarantees and commitmentsCyprus11-04-2011
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);Cyprus11-04-2011
7b. Foreign exchangeCyprus11-04-2011
7c. Financial futures and options;Cyprus11-04-2011
7d. Exchange rate and interest rate instrumentsCyprus11-04-2011
07e. Transferable securitiesCyprus11-04-2011
08. Participation in securities issues and the provision of services related to such issuesCyprus11-04-2011
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsCyprus11-04-2011
11. Portfolio management and adviceCyprus11-04-2011
12. Safekeeping and administration of securitiesCyprus11-04-2011
2:110 Provision of bank services to EEA01. Acceptance of deposits and other repayable fundsCzech Republic01-01-2013
02. LendingCzech Republic01-01-2013
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);Czech Republic01-01-2013
7b. Foreign exchangeCzech Republic01-01-2013
7c. Financial futures and options;Czech Republic01-01-2013
7d. Exchange rate and interest rate instrumentsCzech Republic01-01-2013
07e. Transferable securitiesCzech Republic01-01-2013
08. Participation in securities issues and the provision of services related to such issuesCzech Republic01-01-2013
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsCzech Republic01-01-2013
11. Portfolio management and adviceCzech Republic01-01-2013
12. Safekeeping and administration of securitiesCzech Republic01-01-2013
2:110 Provision of bank services to EEA01. Acceptance of deposits and other repayable fundsDenmark11-04-2011
02. LendingDenmark11-04-2011
06. Guarantees and commitmentsDenmark11-04-2011
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);Denmark11-04-2011
7b. Foreign exchangeDenmark11-04-2011
7c. Financial futures and options;Denmark11-04-2011
7d. Exchange rate and interest rate instrumentsDenmark11-04-2011
07e. Transferable securitiesDenmark11-04-2011
08. Participation in securities issues and the provision of services related to such issuesDenmark11-04-2011
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsDenmark11-04-2011
11. Portfolio management and adviceDenmark11-04-2011
12. Safekeeping and administration of securitiesDenmark11-04-2011
2:110 Provision of bank services to EEAEstonia30-06-2017
2:110 Provision of bank services to EEA01. Acceptance of deposits and other repayable fundsFinland11-04-2011
02. LendingFinland11-04-2011
06. Guarantees and commitmentsFinland11-04-2011
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);Finland11-04-2011
7b. Foreign exchangeFinland11-04-2011
7c. Financial futures and options;Finland11-04-2011
7d. Exchange rate and interest rate instrumentsFinland11-04-2011
07e. Transferable securitiesFinland11-04-2011
08. Participation in securities issues and the provision of services related to such issuesFinland11-04-2011
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsFinland11-04-2011
11. Portfolio management and adviceFinland11-04-2011
12. Safekeeping and administration of securitiesFinland11-04-2011
2:110 Provision of bank services to EEA01. Acceptance of deposits and other repayable fundsFrance11-04-2011
02. LendingFrance11-04-2011
06. Guarantees and commitmentsFrance11-04-2011
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);France11-04-2011
7b. Foreign exchangeFrance11-04-2011
7c. Financial futures and options;France11-04-2011
7d. Exchange rate and interest rate instrumentsFrance11-04-2011
07e. Transferable securitiesFrance11-04-2011
08. Participation in securities issues and the provision of services related to such issuesFrance11-04-2011
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsFrance11-04-2011
11. Portfolio management and adviceFrance11-04-2011
12. Safekeeping and administration of securitiesFrance11-04-2011
2:108 Branch banking to EEA01. Acceptance of deposits and other repayable fundsGermany08-04-2005
02. LendingGermany08-04-2005
03. Financial leasingGermany08-04-2005
04. Payment services as referred to in Article 4(3) of Directive 2015/2366Germany08-04-2005
05. Issuing and administering means of payment (e.g. credit cards, travellers' cheques and bankers' drafts)Germany08-04-2005
06. Guarantees and commitmentsGermany08-04-2005
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);Germany08-04-2005
7b. Foreign exchangeGermany08-04-2005
7c. Financial futures and options;Germany08-04-2005
7d. Exchange rate and interest rate instrumentsGermany08-04-2005
07e. Transferable securitiesGermany08-04-2005
08. Participation in securities issues and the provision of services related to such issuesGermany08-04-2005
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsGermany08-04-2005
10. Money brokingGermany08-04-2005
11. Portfolio management and adviceGermany08-04-2005
12. Safekeeping and administration of securitiesGermany08-04-2005
13. Credit reference servicesGermany08-04-2005
14. Safe custody servicesGermany08-04-2005
2:110 Provision of bank services to EEA01. Acceptance of deposits and other repayable fundsGermany11-04-2011
02. LendingGermany11-04-2011
06. Guarantees and commitmentsGermany11-04-2011
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);Germany11-04-2011
7b. Foreign exchangeGermany11-04-2011
7c. Financial futures and options;Germany11-04-2011
7d. Exchange rate and interest rate instrumentsGermany11-04-2011
07e. Transferable securitiesGermany11-04-2011
08. Participation in securities issues and the provision of services related to such issuesGermany11-04-2011
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsGermany11-04-2011
11. Portfolio management and adviceGermany11-04-2011
12. Safekeeping and administration of securitiesGermany11-04-2011
2:110 Provision of bank services to EEA01. Acceptance of deposits and other repayable fundsGreece11-04-2011
02. LendingGreece11-04-2011
06. Guarantees and commitmentsGreece11-04-2011
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);Greece11-04-2011
7b. Foreign exchangeGreece11-04-2011
7c. Financial futures and options;Greece11-04-2011
7d. Exchange rate and interest rate instrumentsGreece11-04-2011
07e. Transferable securitiesGreece11-04-2011
08. Participation in securities issues and the provision of services related to such issuesGreece11-04-2011
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsGreece11-04-2011
11. Portfolio management and adviceGreece11-04-2011
12. Safekeeping and administration of securitiesGreece11-04-2011
2:110 Provision of bank services to EEAHungary30-06-2017
2:110 Provision of bank services to EEA01. Acceptance of deposits and other repayable fundsIceland11-04-2011
02. LendingIceland11-04-2011
06. Guarantees and commitmentsIceland11-04-2011
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);Iceland11-04-2011
7b. Foreign exchangeIceland11-04-2011
7c. Financial futures and options;Iceland11-04-2011
7d. Exchange rate and interest rate instrumentsIceland11-04-2011
07e. Transferable securitiesIceland11-04-2011
08. Participation in securities issues and the provision of services related to such issuesIceland11-04-2011
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsIceland11-04-2011
11. Portfolio management and adviceIceland11-04-2011
12. Safekeeping and administration of securitiesIceland11-04-2011
2:110 Provision of bank services to EEA01. Acceptance of deposits and other repayable fundsIreland11-04-2011
02. LendingIreland11-04-2011
06. Guarantees and commitmentsIreland11-04-2011
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);Ireland11-04-2011
7b. Foreign exchangeIreland11-04-2011
7c. Financial futures and options;Ireland11-04-2011
7d. Exchange rate and interest rate instrumentsIreland11-04-2011
07e. Transferable securitiesIreland11-04-2011
08. Participation in securities issues and the provision of services related to such issuesIreland11-04-2011
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsIreland11-04-2011
11. Portfolio management and adviceIreland11-04-2011
12. Safekeeping and administration of securitiesIreland11-04-2011
2:110 Provision of bank services to EEA01. Acceptance of deposits and other repayable fundsItaly11-04-2011
02. LendingItaly11-04-2011
06. Guarantees and commitmentsItaly11-04-2011
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);Italy11-04-2011
7b. Foreign exchangeItaly11-04-2011
7c. Financial futures and options;Italy11-04-2011
7d. Exchange rate and interest rate instrumentsItaly11-04-2011
07e. Transferable securitiesItaly11-04-2011
08. Participation in securities issues and the provision of services related to such issuesItaly11-04-2011
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsItaly11-04-2011
11. Portfolio management and adviceItaly11-04-2011
12. Safekeeping and administration of securitiesItaly11-04-2011
2:110 Provision of bank services to EEALatvia30-06-2017
2:110 Provision of bank services to EEA08. Participation in securities issues and the provision of services related to such issuesLiechtenstein11-04-2011
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsLiechtenstein11-04-2011
11. Portfolio management and adviceLiechtenstein11-04-2011
12. Safekeeping and administration of securitiesLiechtenstein11-04-2011
2:110 Provision of bank services to EEALithuania30-06-2017
2:110 Provision of bank services to EEA01. Acceptance of deposits and other repayable fundsLuxembourg11-04-2011
02. LendingLuxembourg11-04-2011
06. Guarantees and commitmentsLuxembourg11-04-2011
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);Luxembourg11-04-2011
7b. Foreign exchangeLuxembourg11-04-2011
7c. Financial futures and options;Luxembourg11-04-2011
7d. Exchange rate and interest rate instrumentsLuxembourg11-04-2011
07e. Transferable securitiesLuxembourg11-04-2011
08. Participation in securities issues and the provision of services related to such issuesLuxembourg11-04-2011
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsLuxembourg11-04-2011
11. Portfolio management and adviceLuxembourg11-04-2011
12. Safekeeping and administration of securitiesLuxembourg11-04-2011
2:110 Provision of bank services to EEAMalta02-06-2015
2:110 Provision of bank services to EEA01. Acceptance of deposits and other repayable fundsNorway11-04-2011
02. LendingNorway11-04-2011
06. Guarantees and commitmentsNorway11-04-2011
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);Norway11-04-2011
7b. Foreign exchangeNorway11-04-2011
7c. Financial futures and options;Norway11-04-2011
7d. Exchange rate and interest rate instrumentsNorway11-04-2011
07e. Transferable securitiesNorway11-04-2011
08. Participation in securities issues and the provision of services related to such issuesNorway11-04-2011
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsNorway11-04-2011
11. Portfolio management and adviceNorway11-04-2011
12. Safekeeping and administration of securitiesNorway11-04-2011
2:110 Provision of bank services to EEAPoland15-05-2012
2:110 Provision of bank services to EEA01. Acceptance of deposits and other repayable fundsPortugal11-04-2011
02. LendingPortugal11-04-2011
06. Guarantees and commitmentsPortugal11-04-2011
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);Portugal11-04-2011
7b. Foreign exchangePortugal11-04-2011
7c. Financial futures and options;Portugal11-04-2011
7d. Exchange rate and interest rate instrumentsPortugal11-04-2011
07e. Transferable securitiesPortugal11-04-2011
08. Participation in securities issues and the provision of services related to such issuesPortugal11-04-2011
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsPortugal11-04-2011
11. Portfolio management and advicePortugal11-04-2011
12. Safekeeping and administration of securitiesPortugal11-04-2011
2:110 Provision of bank services to EEARomania30-06-2017
2:110 Provision of bank services to EEASlovakia30-06-2017
2:110 Provision of bank services to EEASlovenia30-06-2017
2:110 Provision of bank services to EEA01. Acceptance of deposits and other repayable fundsSpain11-04-2011
02. LendingSpain11-04-2011
06. Guarantees and commitmentsSpain11-04-2011
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);Spain11-04-2011
7b. Foreign exchangeSpain11-04-2011
7c. Financial futures and options;Spain11-04-2011
7d. Exchange rate and interest rate instrumentsSpain11-04-2011
07e. Transferable securitiesSpain11-04-2011
08. Participation in securities issues and the provision of services related to such issuesSpain11-04-2011
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsSpain11-04-2011
11. Portfolio management and adviceSpain11-04-2011
12. Safekeeping and administration of securitiesSpain11-04-2011
2:110 Provision of bank services to EEA01. Acceptance of deposits and other repayable fundsSweden11-04-2011
02. LendingSweden11-04-2011
06. Guarantees and commitmentsSweden11-04-2011
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);Sweden11-04-2011
7b. Foreign exchangeSweden11-04-2011
7c. Financial futures and options;Sweden11-04-2011
7d. Exchange rate and interest rate instrumentsSweden11-04-2011
07e. Transferable securitiesSweden11-04-2011
08. Participation in securities issues and the provision of services related to such issuesSweden11-04-2011
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsSweden11-04-2011
11. Portfolio management and adviceSweden11-04-2011
12. Safekeeping and administration of securitiesSweden11-04-2011
2:108 Branch banking to EEA02. LendingUnited Kingdom29-09-1999
06. Guarantees and commitmentsUnited Kingdom29-09-1999
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);United Kingdom29-09-1999
7b. Foreign exchangeUnited Kingdom29-09-1999
7c. Financial futures and options;United Kingdom29-09-1999
7d. Exchange rate and interest rate instrumentsUnited Kingdom29-09-1999
07e. Transferable securitiesUnited Kingdom29-09-1999
08. Participation in securities issues and the provision of services related to such issuesUnited Kingdom29-09-1999
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsUnited Kingdom29-09-1999
11. Portfolio management and adviceUnited Kingdom29-09-1999
12. Safekeeping and administration of securitiesUnited Kingdom29-09-1999
2:110 Provision of bank services to EEA02. LendingUnited Kingdom11-04-2011
06. Guarantees and commitmentsUnited Kingdom11-04-2011
7a. Money market instruments (cheques, bills, certificates of deposit, etc.);United Kingdom11-04-2011
7b. Foreign exchangeUnited Kingdom11-04-2011
7c. Financial futures and options;United Kingdom11-04-2011
7d. Exchange rate and interest rate instrumentsUnited Kingdom11-04-2011
07e. Transferable securitiesUnited Kingdom11-04-2011
08. Participation in securities issues and the provision of services related to such issuesUnited Kingdom11-04-2011
09. Advice to undertakings on capital structure, corporate strategy and related questions and advice as well as services relating to mergers and the purchase of undertakingsUnited Kingdom11-04-2011
11. Portfolio management and adviceUnited Kingdom11-04-2011
12. Safekeeping and administration of securitiesUnited Kingdom11-04-2011
+
+
+
+
+
+

Covered Bonds

+
+ + + + + + + + + + + +
Name covered bonds program van gedekte obligatiesIssuing date offering prospectus of the programDate of registrationIn accordance with article 129 (1) of Regulation (EU) Nr 575/2013 (CRR)Remarks
Conditional Pass-Through Covered Bond Progr. EUR 5.000.000.00022-07-201309-09-2013jaClick + + here + + to find the investor reports. (You will be redirected to the issuer's website.) +
+
+
+
+
+Publication Date : + 29-05-2019 +
+ + + + + + + + + + +
+ + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Print +
+ + +
+ Download +
+ +
+
+ + + + + +
+ + + + + + + + +
+ + + +
+ + + +
+ + +
+ + + + diff --git a/tests/data/nl/din329_d2_01.json b/tests/data/nl/din329_d2_01.json new file mode 100644 index 0000000..9039836 --- /dev/null +++ b/tests/data/nl/din329_d2_01.json @@ -0,0 +1,42 @@ +[ + [ + "Statutory name", + "NIBC Bank N.V." + ], + [ + "Trade name", + "NIBC Direct" + ], + [ + "Address", + "Carnegieplein 4" + ], + [ + "Postal code", + "2517 KJ" + ], + [ + "Place of residence", + "'S-GRAVENHAGE" + ], + [ + "Country", + "Netherlands" + ], + [ + "Disclosure", + "" + ], + [ + "Chamber of Commerce", + "27032036" + ], + [ + "Relation number DNB", + "B0685" + ], + [ + "Category", + "Emittent effecten CSDB, Bank" + ] +] diff --git a/tests/data/no/cb_001_ps.html b/tests/data/no/cb_001_ps.html new file mode 100644 index 0000000..c471156 --- /dev/null +++ b/tests/data/no/cb_001_ps.html @@ -0,0 +1,21 @@ +

Cross-border services/classes

+

+ The entity may provide cross-border services/classes from Slovakia +

+ + +
+

To Norway without physical establishment, for the following services/classes

+
    +
  • + Payment service activity +
      +
    • + Service 3 and 4 in annex to PSD 1 +
      • +
    +
    • +
+ + +
diff --git a/tests/data/no/cb_001_ps.json b/tests/data/no/cb_001_ps.json new file mode 100644 index 0000000..9323319 --- /dev/null +++ b/tests/data/no/cb_001_ps.json @@ -0,0 +1,16 @@ +{ + "name": "Cross-border services/classes The entity may provide cross-border services/classes from Slovakia", + "data": [ + { + "name": "To Norway without physical establishment, for the following services/classes", + "data": [ + { + "name": "Payment service activity", + "data": [ + "Service 3 and 4 in annex to PSD 1" + ] + } + ] + } + ] +} diff --git a/tests/data/no/cb_002_ps.html b/tests/data/no/cb_002_ps.html new file mode 100644 index 0000000..cdcea25 --- /dev/null +++ b/tests/data/no/cb_002_ps.html @@ -0,0 +1,36 @@ +

Cross-border services/classes

+

+ The entity may provide cross-border services/classes from United Kingdom +

+ + +
+

To Norway without physical establishment, for the following services/classes

+
    +
  • + Payment service activity +
      +
    • + Service 6 in annex to PSD 1 +
      • +
    +
    • +
+ + +
+ + + + +
+

To Norway through tied agents

+ +
diff --git a/tests/data/no/cb_002_ps.json b/tests/data/no/cb_002_ps.json new file mode 100644 index 0000000..eab577f --- /dev/null +++ b/tests/data/no/cb_002_ps.json @@ -0,0 +1,23 @@ +{ + "name": "Cross-border services/classes The entity may provide cross-border services/classes from United Kingdom", + "data": [ + { + "name": "To Norway without physical establishment, for the following services/classes", + "data": [ + { + "name": "Payment service activity", + "data": [ + "Service 6 in annex to PSD 1" + ] + } + ] + }, + { + "name": "To Norway through tied agents", + "data": [ + "HASHYL FINANCIAL SERVICES YAHYA SONKO (Norway)", + "SHARANHA GULLSMED Varathalingam Selliah (Norway)" + ] + } + ] +} diff --git a/tests/data/no/cb_003_ps.html b/tests/data/no/cb_003_ps.html new file mode 100644 index 0000000..8653d90 --- /dev/null +++ b/tests/data/no/cb_003_ps.html @@ -0,0 +1,24 @@ +

Cross-border services/classes

+

+ The entity may provide cross-border services/classes from United Kingdom +

+ + +
+

To Norway without physical establishment, for the following services/classes

+
    +
  • + Payment service activity +
      +
    • + Service 3 and 4 in annex to PSD 1 +
      • +
    • + Service 5 in annex to PSD 1 +
      • +
    +
    • +
+ + +
diff --git a/tests/data/no/cb_003_ps.json b/tests/data/no/cb_003_ps.json new file mode 100644 index 0000000..da8a19b --- /dev/null +++ b/tests/data/no/cb_003_ps.json @@ -0,0 +1,17 @@ +{ + "name": "Cross-border services/classes The entity may provide cross-border services/classes from United Kingdom", + "data": [ + { + "name": "To Norway without physical establishment, for the following services/classes", + "data": [ + { + "name": "Payment service activity", + "data": [ + "Service 3 and 4 in annex to PSD 1", + "Service 5 in annex to PSD 1" + ] + } + ] + } + ] +} diff --git a/tests/data/no/cb_004_ps.html b/tests/data/no/cb_004_ps.html new file mode 100644 index 0000000..2960b2c --- /dev/null +++ b/tests/data/no/cb_004_ps.html @@ -0,0 +1,33 @@ +

Cross-border services/classes

+

+ The entity may provide cross-border services/classes from United Kingdom +

+ + +
+

To Norway without physical establishment, for the following services/classes

+
    +
  • + Payment service activity +
      +
    • + Service 1 and 2 in annex to PSD 1 +
      • +
    • + Service 3 and 4 in annex to PSD 1 +
      • +
    • + Service 5 in annex to PSD 1 +
      • +
    • + Service 6 in annex to PSD 1 +
      • +
    • + Service 7 in annex to PSD 1 +
      • +
    +
    • +
+ + +
diff --git a/tests/data/no/cb_004_ps.json b/tests/data/no/cb_004_ps.json new file mode 100644 index 0000000..90ee92d --- /dev/null +++ b/tests/data/no/cb_004_ps.json @@ -0,0 +1,20 @@ +{ + "name": "Cross-border services/classes The entity may provide cross-border services/classes from United Kingdom", + "data": [ + { + "name": "To Norway without physical establishment, for the following services/classes", + "data": [ + { + "name": "Payment service activity", + "data": [ + "Service 1 and 2 in annex to PSD 1", + "Service 3 and 4 in annex to PSD 1", + "Service 5 in annex to PSD 1", + "Service 6 in annex to PSD 1", + "Service 7 in annex to PSD 1" + ] + } + ] + } + ] +} diff --git a/tests/data/no/cb_005_ps_with_empty_item.html b/tests/data/no/cb_005_ps_with_empty_item.html new file mode 100644 index 0000000..18b9274 --- /dev/null +++ b/tests/data/no/cb_005_ps_with_empty_item.html @@ -0,0 +1,32 @@ +

Cross-border services/classes

+

+ The entity may provide cross-border services/classes from Germany +

+ + +
+

To Norway without physical establishment, for the following services/classes

+
    +
  • + +
      +
    • + Distribution/Redemption of electronic money +
      • +
    • + Issuing electronic money +
      • +
    +
    • +
  • + Payment service activity +
      +
    • + Service 5 in annex to PSD 1 +
      • +
    +
    • +
+ + +
diff --git a/tests/data/no/cb_005_ps_with_empty_item.json b/tests/data/no/cb_005_ps_with_empty_item.json new file mode 100644 index 0000000..b2ac62d --- /dev/null +++ b/tests/data/no/cb_005_ps_with_empty_item.json @@ -0,0 +1,23 @@ +{ + "name": "Cross-border services/classes The entity may provide cross-border services/classes from Germany", + "data": [ + { + "name": "To Norway without physical establishment, for the following services/classes", + "data": [ + { + "name": "", + "data": [ + "Distribution/Redemption of electronic money", + "Issuing electronic money" + ] + }, + { + "name": "Payment service activity", + "data": [ + "Service 5 in annex to PSD 1" + ] + } + ] + } + ] +} diff --git a/tests/data/no/cb_006_em_with_multi_countries.html b/tests/data/no/cb_006_em_with_multi_countries.html new file mode 100644 index 0000000..f599ef7 --- /dev/null +++ b/tests/data/no/cb_006_em_with_multi_countries.html @@ -0,0 +1,71 @@ +

Cross-border services/classes

+

+ The entity may provide cross-border services/classes from Norway +

+ + +
+

To multiple countries without physical establishment, for the following services/classes

+
    +
  • + +
      +
    • + Distribution/Redemption of electronic money +
        +
      • + Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom +
        • +
      +
      • +
    • + Issuing electronic money +
        +
      • + Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom +
        • +
      +
      • +
    +
    • +
  • + Payment service activity +
      +
    • + Service 1 and 2 in annex to PSD 1 +
        +
      • + Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom +
        • +
      +
      • +
    • + Service 3 and 4 in annex to PSD 1 +
        +
      • + Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom +
        • +
      +
      • +
    • + Service 5 in annex to PSD 1 +
        +
      • + Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom +
        • +
      +
      • +
    • + Service 6 in annex to PSD 1 +
        +
      • + Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom +
        • +
      +
      • +
    +
    • +
+ + +
diff --git a/tests/data/no/cb_006_em_with_multi_countries.json b/tests/data/no/cb_006_em_with_multi_countries.json new file mode 100644 index 0000000..4c7c257 --- /dev/null +++ b/tests/data/no/cb_006_em_with_multi_countries.json @@ -0,0 +1,44 @@ +{ + "name": "Cross-border services/classes The entity may provide cross-border services/classes from Norway", + "data": [ + { + "name": "To multiple countries without physical establishment, for the following services/classes", + "data": [ + { + "name": "", + "data": [ + { + "name": "Distribution/Redemption of electronic money", + "data": [ "Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom" ] + }, + { + "name": "Issuing electronic money", + "data": [ "Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom" ] + } + ] + }, + { + "name": "Payment service activity", + "data": [ + { + "name": "Service 1 and 2 in annex to PSD 1", + "data": [ "Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom" ] + }, + { + "name": "Service 3 and 4 in annex to PSD 1", + "data": [ "Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom" ] + }, + { + "name": "Service 5 in annex to PSD 1", + "data": [ "Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom" ] + }, + { + "name": "Service 6 in annex to PSD 1", + "data": [ "Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom" ] + } + ] + } + ] + } + ] +} diff --git a/tests/data/no/ent_002_cb.json b/tests/data/no/ent_002_cb.json index abe26fb..ea27420 100644 --- a/tests/data/no/ent_002_cb.json +++ b/tests/data/no/ent_002_cb.json @@ -1,45 +1,50 @@ { - "crossborderServicesclasses": [ - "The entity may provide cross-border services/classes from Spain" - ], + "name": "Cross-border services/classes The entity may provide cross-border services/classes from Spain", "data": [ { - "description": { - "main": "main", - "ancillaryServicesMiFidDirective": "Ancillary services (MiFid directive)", - "investmentServicesAndActivitiesMiFidDirective": "Investment services and activities (MiFid directive)" - }, - "main": [ - "1. Acceptance of deposits and other repayable funds", - "2. Lending", - "3. Financial leasing", - "6. Guarantees and commitments", - "7. a) Money-market instruments", - "7. b) Foreign exchange", - "7. c) Financial future/forward contracts and options", - "7. d) Foreign exchange- and interest rate instruments", - "7. e) Transferable securities", - "8. Participation in securities issues and the provision of services related to such issues", - "9. Advisory services", - "11. Portfolio management and advice" - ], - "ancillaryServicesMiFidDirective": [ - "No 1: Safekeeping and administration of financial instruments", - "No 2: Granting credits or loans", - "No 3: Advice on capital structure, strategy et al.; advice and services re. to mergers and purchases", - "No 4: Foreign exchange services connected to the provision of investment services", - "No 5: Provision of investment recommendations, analyses and general recommendations", - "No 6: Services related to underwriting", - "No 7: Services related to underlying of derivatives connected to investment and ancillary services" - ], - "investmentServicesAndActivitiesMiFidDirective": [ - "No 1: Reception and transmission of orders", - "No 2: Execution of orders", - "No 3: Dealing on own account", - "No 4: Active management of investors' portfolios", - "No 5: Investment advice", - "No 6: Underwriting of fin. instruments or placing of fin. instruments on a firm commitment basis", - "No 7: Placing of financial instruments without a firm commitment basis" + "name": "To Norway without physical establishment, for the following services/classes", + "data": [ + { + "name": "", + "data": [ + "1. Acceptance of deposits and other repayable funds", + "2. Lending", + "3. Financial leasing", + "6. Guarantees and commitments", + "7. a) Money-market instruments", + "7. b) Foreign exchange", + "7. c) Financial future/forward contracts and options", + "7. d) Foreign exchange- and interest rate instruments", + "7. e) Transferable securities", + "8. Participation in securities issues and the provision of services related to such issues", + "9. Advisory services", + "11. Portfolio management and advice" + ] + }, + { + "name": "Ancillary services (MiFid directive)", + "data": [ + "No 1: Safekeeping and administration of financial instruments", + "No 2: Granting credits or loans", + "No 3: Advice on capital structure, strategy et al.; advice and services re. to mergers and purchases", + "No 4: Foreign exchange services connected to the provision of investment services", + "No 5: Provision of investment recommendations, analyses and general recommendations", + "No 6: Services related to underwriting", + "No 7: Services related to underlying of derivatives connected to investment and ancillary services" + ] + }, + { + "name": "Investment services and activities (MiFid directive)", + "data": [ + "No 1: Reception and transmission of orders", + "No 2: Execution of orders", + "No 3: Dealing on own account", + "No 4: Active management of investors' portfolios", + "No 5: Investment advice", + "No 6: Underwriting of fin. instruments or placing of fin. instruments on a firm commitment basis", + "No 7: Placing of financial instruments without a firm commitment basis" + ] + } ] } ] diff --git a/tests/data/no/ent_002_sniff.json b/tests/data/no/ent_002_sniff.json index f651c98..1667e0b 100644 --- a/tests/data/no/ent_002_sniff.json +++ b/tests/data/no/ent_002_sniff.json @@ -12,47 +12,52 @@ "1/1/2016" ], "crossBorder": { - "crossborderServicesclasses": [ - "The entity may provide cross-border services/classes from Spain" - ], + "name": "Cross-border services/classes The entity may provide cross-border services/classes from Spain", "data": [ { - "description": { - "main": "main", - "ancillaryServicesMiFidDirective": "Ancillary services (MiFid directive)", - "investmentServicesAndActivitiesMiFidDirective": "Investment services and activities (MiFid directive)" - }, - "main": [ - "1. Acceptance of deposits and other repayable funds", - "2. Lending", - "3. Financial leasing", - "6. Guarantees and commitments", - "7. a) Money-market instruments", - "7. b) Foreign exchange", - "7. c) Financial future/forward contracts and options", - "7. d) Foreign exchange- and interest rate instruments", - "7. e) Transferable securities", - "8. Participation in securities issues and the provision of services related to such issues", - "9. Advisory services", - "11. Portfolio management and advice" - ], - "ancillaryServicesMiFidDirective": [ - "No 1: Safekeeping and administration of financial instruments", - "No 2: Granting credits or loans", - "No 3: Advice on capital structure, strategy et al.; advice and services re. to mergers and purchases", - "No 4: Foreign exchange services connected to the provision of investment services", - "No 5: Provision of investment recommendations, analyses and general recommendations", - "No 6: Services related to underwriting", - "No 7: Services related to underlying of derivatives connected to investment and ancillary services" - ], - "investmentServicesAndActivitiesMiFidDirective": [ - "No 1: Reception and transmission of orders", - "No 2: Execution of orders", - "No 3: Dealing on own account", - "No 4: Active management of investors' portfolios", - "No 5: Investment advice", - "No 6: Underwriting of fin. instruments or placing of fin. instruments on a firm commitment basis", - "No 7: Placing of financial instruments without a firm commitment basis" + "name": "To Norway without physical establishment, for the following services/classes", + "data": [ + { + "name": "", + "data": [ + "1. Acceptance of deposits and other repayable funds", + "2. Lending", + "3. Financial leasing", + "6. Guarantees and commitments", + "7. a) Money-market instruments", + "7. b) Foreign exchange", + "7. c) Financial future/forward contracts and options", + "7. d) Foreign exchange- and interest rate instruments", + "7. e) Transferable securities", + "8. Participation in securities issues and the provision of services related to such issues", + "9. Advisory services", + "11. Portfolio management and advice" + ] + }, + { + "name": "Ancillary services (MiFid directive)", + "data": [ + "No 1: Safekeeping and administration of financial instruments", + "No 2: Granting credits or loans", + "No 3: Advice on capital structure, strategy et al.; advice and services re. to mergers and purchases", + "No 4: Foreign exchange services connected to the provision of investment services", + "No 5: Provision of investment recommendations, analyses and general recommendations", + "No 6: Services related to underwriting", + "No 7: Services related to underlying of derivatives connected to investment and ancillary services" + ] + }, + { + "name": "Investment services and activities (MiFid directive)", + "data": [ + "No 1: Reception and transmission of orders", + "No 2: Execution of orders", + "No 3: Dealing on own account", + "No 4: Active management of investors' portfolios", + "No 5: Investment advice", + "No 6: Underwriting of fin. instruments or placing of fin. instruments on a firm commitment basis", + "No 7: Placing of financial instruments without a firm commitment basis" + ] + } ] } ] diff --git a/tests/data/no/ent_004_sniff.json b/tests/data/no/ent_004_sniff.json index 612f649..a6830b2 100644 --- a/tests/data/no/ent_004_sniff.json +++ b/tests/data/no/ent_004_sniff.json @@ -11,49 +11,55 @@ "registrationDate": [ "1/1/2016" ], + "crossBorder": { - "crossborderServicesclasses": [ - "The entity may provide cross-border services/classes from Luxembourg" - ], + "name": "Cross-border services/classes The entity may provide cross-border services/classes from Luxembourg", "data": [ { - "description": { - "main": "main", - "ancillaryServicesMiFidDirective": "Ancillary services (MiFid directive)", - "investmentServicesAndActivitiesMiFidDirective": "Investment services and activities (MiFid directive)" - }, - "main": [ - "1. Acceptance of deposits and other repayable funds", - "2. Lending", - "3. Financial leasing", - "4. Money transmission services", - "5. Issuing and administering of payments (credit cards etc.)", - "6. Guarantees and commitments", - "7. a) Money-market instruments", - "7. b) Foreign exchange", - "7. c) Financial future/forward contracts and options", - "7. d) Foreign exchange- and interest rate instruments", - "7. e) Transferable securities", - "8. Participation in securities issues and the provision of services related to such issues", - "9. Advisory services", - "10. Money broking", - "11. Portfolio management and advice", - "12. Safekeeping and administration of securities", - "13. Credit reference services", - "14. Safe custody services" - ], - "ancillaryServicesMiFidDirective": [ - "No 1: Safekeeping and administration of financial instruments", - "No 2: Granting credits or loans", - "No 4: Foreign exchange services connected to the provision of investment services", - "No 5: Provision of investment recommendations, analyses and general recommendations", - "No 7: Services related to underlying of derivatives connected to investment and ancillary services" - ], - "investmentServicesAndActivitiesMiFidDirective": [ - "No 1: Reception and transmission of orders", - "No 2: Execution of orders", - "No 4: Active management of investors' portfolios", - "No 5: Investment advice" + "name": "To Norway without physical establishment, for the following services/classes", + "data": [ + { + "name": "", + "data": [ + "1. Acceptance of deposits and other repayable funds", + "2. Lending", + "3. Financial leasing", + "4. Money transmission services", + "5. Issuing and administering of payments (credit cards etc.)", + "6. Guarantees and commitments", + "7. a) Money-market instruments", + "7. b) Foreign exchange", + "7. c) Financial future/forward contracts and options", + "7. d) Foreign exchange- and interest rate instruments", + "7. e) Transferable securities", + "8. Participation in securities issues and the provision of services related to such issues", + "9. Advisory services", + "10. Money broking", + "11. Portfolio management and advice", + "12. Safekeeping and administration of securities", + "13. Credit reference services", + "14. Safe custody services" + ] + }, + { + "name": "Ancillary services (MiFid directive)", + "data": [ + "No 1: Safekeeping and administration of financial instruments", + "No 2: Granting credits or loans", + "No 4: Foreign exchange services connected to the provision of investment services", + "No 5: Provision of investment recommendations, analyses and general recommendations", + "No 7: Services related to underlying of derivatives connected to investment and ancillary services" + ] + }, + { + "name": "Investment services and activities (MiFid directive)", + "data": [ + "No 1: Reception and transmission of orders", + "No 2: Execution of orders", + "No 4: Active management of investors' portfolios", + "No 5: Investment advice" + ] + } ] } ] diff --git a/tests/data/no/ent_005_cb.json b/tests/data/no/ent_005_cb.json index 3b0b46a..ed036ad 100644 --- a/tests/data/no/ent_005_cb.json +++ b/tests/data/no/ent_005_cb.json @@ -1,13 +1,9 @@ { - "crossborderServicesclasses": [ - "The entity may provide cross-border services/classes from Sweden" - ], + "name": "Cross-border services/classes The entity may provide cross-border services/classes from Sweden", "data": [ { - "description": { - "toNorwayThroughABranchEstablishment": "To Norway through a branch establishment" - }, - "toNorwayThroughABranchEstablishment": [ + "name": "To Norway through a branch establishment", + "data": [ "AVIDA FINANS AB NUF (Norway)" ] } diff --git a/tests/data/no/ent_005_sniff.json b/tests/data/no/ent_005_sniff.json index 2918949..3aba9ca 100644 --- a/tests/data/no/ent_005_sniff.json +++ b/tests/data/no/ent_005_sniff.json @@ -12,19 +12,16 @@ "8/4/2008" ], "crossBorder": { - "crossborderServicesclasses": [ - "The entity may provide cross-border services/classes from Sweden" - ], + "name": "Cross-border services/classes The entity may provide cross-border services/classes from Sweden", "data": [ { - "description": { - "toNorwayThroughABranchEstablishment": "To Norway through a branch establishment" - }, - "toNorwayThroughABranchEstablishment": [ + "name": "To Norway through a branch establishment", + "data": [ "AVIDA FINANS AB NUF (Norway)" ] } ] } + } ] diff --git a/tests/data/no/ent_006_cb.json b/tests/data/no/ent_006_cb.json index 585f4df..eab577f 100644 --- a/tests/data/no/ent_006_cb.json +++ b/tests/data/no/ent_006_cb.json @@ -1,21 +1,20 @@ { - "crossborderServicesclasses": [ - "The entity may provide cross-border services/classes from United Kingdom" - ], + "name": "Cross-border services/classes The entity may provide cross-border services/classes from United Kingdom", "data": [ { - "description": { - "paymentServiceActivity": "Payment service activity" - }, - "paymentServiceActivity": [ - "Service 6 in annex to PSD 1" + "name": "To Norway without physical establishment, for the following services/classes", + "data": [ + { + "name": "Payment service activity", + "data": [ + "Service 6 in annex to PSD 1" + ] + } ] }, { - "description": { - "toNorwayThroughTiedAgents": "To Norway through tied agents" - }, - "toNorwayThroughTiedAgents": [ + "name": "To Norway through tied agents", + "data": [ "HASHYL FINANCIAL SERVICES YAHYA SONKO (Norway)", "SHARANHA GULLSMED Varathalingam Selliah (Norway)" ] diff --git a/tests/data/no/ent_006_sniff.json b/tests/data/no/ent_006_sniff.json index 94e0ded..5845495 100644 --- a/tests/data/no/ent_006_sniff.json +++ b/tests/data/no/ent_006_sniff.json @@ -12,23 +12,22 @@ "1/1/2016" ], "crossBorder": { - "crossborderServicesclasses": [ - "The entity may provide cross-border services/classes from United Kingdom" - ], + "name": "Cross-border services/classes The entity may provide cross-border services/classes from United Kingdom", "data": [ { - "description": { - "paymentServiceActivity": "Payment service activity" - }, - "paymentServiceActivity": [ - "Service 6 in annex to PSD 1" + "name": "To Norway without physical establishment, for the following services/classes", + "data": [ + { + "name": "Payment service activity", + "data": [ + "Service 6 in annex to PSD 1" + ] + } ] }, { - "description": { - "toNorwayThroughTiedAgents": "To Norway through tied agents" - }, - "toNorwayThroughTiedAgents": [ + "name": "To Norway through tied agents", + "data": [ "HASHYL FINANCIAL SERVICES YAHYA SONKO (Norway)", "SHARANHA GULLSMED Varathalingam Selliah (Norway)" ] diff --git a/tests/scrape.at.js b/tests/scrape.at.js new file mode 100644 index 0000000..4f10f0c --- /dev/null +++ b/tests/scrape.at.js @@ -0,0 +1,52 @@ +const tape = require('tape'); +const _test = require('tape-promise').default; // <---- notice 'default' +const test = _test(tape); // decorate tape + +const diff = require('deep-diff'); +const fs = require('fs'); +const jsonfile = require('jsonfile'); + +const Austria = require('../ncas/at'); + +const atScraper = new Austria(); + +test.test('Austria 🇦🇹 :: Tests', async t => { + + t.test('Test extractEntityDetails ent_001', async t => { + const html = fs.readFileSync('tests/data/at/ent_001.html'); + const output = await atScraper.extractEntityDetails(html); + const expectedJSON = jsonfile.readFileSync('tests/data/at/ent_001.json'); + + t.deepEquals(output, expectedJSON, 'Extracted entity details from Page'); + + console.log(diff(output, expectedJSON)); + + t.end(); + }); + + t.test('Test extractEntityDetails ent_002', async t => { + const html = fs.readFileSync('tests/data/at/ent_002.html'); + const output = await atScraper.extractEntityDetails(html); + const expectedJSON = jsonfile.readFileSync('tests/data/at/ent_002.json'); + + t.deepEquals(output, expectedJSON, 'Extracted entity details from Page'); + + console.log(diff(output, expectedJSON)); + + t.end(); + }); + + t.test('Test extractEntityDetails ent_003', async t => { + const html = fs.readFileSync('tests/data/at/ent_003.html'); + const output = await atScraper.extractEntityDetails(html); + const expectedJSON = jsonfile.readFileSync('tests/data/at/ent_003.json'); + + t.deepEquals(output, expectedJSON, 'Extracted entity details from Page'); + + console.log(diff(output, expectedJSON)); + + t.end(); + }); + + t.end(); +}); diff --git a/tests/scrape.be.js b/tests/scrape.be.js new file mode 100644 index 0000000..4311e3b --- /dev/null +++ b/tests/scrape.be.js @@ -0,0 +1,166 @@ +const cheerio = require('cheerio'); +const tape = require('tape'); +const _test = require('tape-promise').default; // <---- notice 'default' +const test = _test(tape); // decorate tape + +const fs = require('fs'); +const jsonfile = require('jsonfile'); + +const Belgium = require('../ncas/be'); +const beScraper = new Belgium(); + +test.test('Entities', async t => { + t.test('Extract main details...', async t => { + t.test('...from td container', async t => { + const htmlFile = 'tests/data/be/ps_fullpage_001.html'; + const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' }); + const $ = cheerio.load(html, { 'decodeEntities': false }); + const detailsContainer = $('ul.List1 div.table-responsive tbody tr td').eq(0); + + const output = await beScraper.extractMainDetails(detailsContainer); + + const expectedJSON = jsonfile.readFileSync('tests/data/be/ps_001_mainDetails.json'); + t.deepEquals(output, expectedJSON); + + t.end(); + }); + + t.test('...from li container', async t => { + const htmlFile = 'tests/data/be/ci_fullpage_001.html'; + const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' }); + const $ = cheerio.load(html, { 'decodeEntities': false }); + const detailsContainer = $('ul.List1 ul.List2 > li > ul > li').eq(0); + + const output = await beScraper.extractMainDetails(detailsContainer); + + const expectedJSON = jsonfile.readFileSync('tests/data/be/ci_001_mainDetails.json'); + t.deepEquals(output, expectedJSON); + + t.end(); + }); + + t.test('...from unusual entity (3-line address and large spacing)', async t => { + const htmlFile = 'tests/data/be/em_fullpage_001.html'; + const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' }); + const $ = cheerio.load(html, { 'decodeEntities': false }); + const detailsContainer = $('ul.List1 div.table-responsive tbody tr').eq(4).children('td').eq(0); + + const output = await beScraper.extractMainDetails(detailsContainer); + + const expectedJSON = jsonfile.readFileSync('tests/data/be/em_001_mainDetails.json'); + t.deepEquals(output, expectedJSON); + + t.end(); + }); + + t.end(); + }); + + t.test('Extract full details...', async t => { + t.test('...from payment service', async t => { + const htmlFile = 'tests/data/be/ps_fullpage_001.html'; + const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' }); + const $ = cheerio.load(html, { 'decodeEntities': false }); + const fullDetailsContainer = $('ul.List1 div.table-responsive tbody tr').eq(0); + + const output = await beScraper.extractFullDetails(fullDetailsContainer, 0); + + const expectedJSON = jsonfile.readFileSync('tests/data/be/ps_001_fullDetails.json'); + t.deepEquals(output, expectedJSON); + + t.end(); + }); + + t.test('...from emoney service', async t => { + const htmlFile = 'tests/data/be/em_fullpage_001.html'; + const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' }); + const $ = cheerio.load(html, { 'decodeEntities': false }); + const fullDetailsContainer = $('ul.List1 div.table-responsive tbody tr').eq(0); + + const output = await beScraper.extractFullDetails(fullDetailsContainer, 0); + + const expectedJSON = jsonfile.readFileSync('tests/data/be/em_001_fullDetails.json'); + t.deepEquals(output, expectedJSON); + + t.end(); + }); + + t.test('...from credit institution', async t => { + const htmlFile = 'tests/data/be/ci_fullpage_001.html'; + const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' }); + const $ = cheerio.load(html, { 'decodeEntities': false }); + const fullDetailsContainer = $('ul.List1 ul.List2 > li > ul > li').eq(0); + + const output = await beScraper.extractFullDetails(fullDetailsContainer, 2); + + const expectedJSON = jsonfile.readFileSync('tests/data/be/ci_001_fullDetails.json'); + t.deepEquals(output, expectedJSON); + + t.end(); + }); + + t.end(); + }); + + t.test('Extract entities from container...', async t => { + t.test('...of payment services (tbody)', async t => { + const htmlFile = 'tests/data/be/ps_fullpage_001.html'; + const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' }); + const $ = cheerio.load(html, { 'decodeEntities': false }); + const entitiesContainer = $('ul.List1 tbody'); + + const output = await beScraper.extractEntitiesFromContainer(entitiesContainer, 0); + + t.equals(output.length, 24); + + t.end(); + }); + + t.test('...of credit institutions (ul)', async t => { + const htmlFile = 'tests/data/be/ci_fullpage_001.html'; + const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' }); + const $ = cheerio.load(html, { 'decodeEntities': false }); + const entitiesContainer = $('ul.List1 ul.List2 > li > ul').eq(0); // get the first list only for this test + + const output = await beScraper.extractEntitiesFromContainer(entitiesContainer, 2); + + t.equals(output.length, 25); + + t.end(); + }); + + t.end(); + }); + + t.test('Extract index...', async t => { + t.test('...of payment services', async t => { + const htmlFile = 'tests/data/be/ps_fullpage_001.html'; + const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' }); + const $ = cheerio.load(html, { 'decodeEntities': false }); + const indexContainer = $('#PrudentialList'); + + const output = await beScraper.extractIndex(indexContainer, 0); + + // console.log(output); + + t.end(); + }); + + t.test('...of credit institutions', async t => { + const htmlFile = 'tests/data/be/ci_fullpage_001.html'; + const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' }); + const $ = cheerio.load(html, { 'decodeEntities': false }); + const indexContainer = $('#PrudentialList'); + + const output = await beScraper.extractIndex(indexContainer, 2); + + // console.log(output); + + t.end(); + }); + + t.end(); + }); + + t.end(); +}); diff --git a/tests/scrape.fr.js b/tests/scrape.fr.js index 3e3ca6e..e1e0e08 100644 --- a/tests/scrape.fr.js +++ b/tests/scrape.fr.js @@ -98,7 +98,7 @@ test('FRANCE:: Scrape Indexes', async t => { t.end(); }); - t.end(); + t.end(); }); test('FRANCE Scrape a Payment Instititute', async t => { @@ -222,61 +222,54 @@ test('FRANCE Breaking CI 001', async t => { }); test('FRANCE:: DIR-3741', async t => { - const dir3741_001 = fs.readFileSync('tests/data/fr/dir3741_001.html'); - const dir3741_001Data = jsonfile.readFileSync('tests/data/fr/dir3741_001.json'); + const dir3741001 = fs.readFileSync('tests/data/fr/dir3741_001.html'); + const dir3741001Data = jsonfile.readFileSync('tests/data/fr/dir3741_001.json'); - const dir3741_002 = fs.readFileSync('tests/data/fr/dir3741_002.html'); - const dir3741_002Data = jsonfile.readFileSync('tests/data/fr/dir3741_002.json'); + const dir3741002 = fs.readFileSync('tests/data/fr/dir3741_002.html'); + const dir3741002Data = jsonfile.readFileSync('tests/data/fr/dir3741_002.json'); - const dir3741_003 = fs.readFileSync('tests/data/fr/dir3741_003.html'); - const dir3741_003Data = jsonfile.readFileSync('tests/data/fr/dir3741_003.json'); + const dir3741003 = fs.readFileSync('tests/data/fr/dir3741_003.html'); + const dir3741003Data = jsonfile.readFileSync('tests/data/fr/dir3741_003.json'); const frScraper = new France(); t.test('FRANCE::Extract Details from Page 1/103', async t => { - const $ = cheerio.load(dir3741_001); - const $table = $('table.table tr'); + const $ = cheerio.load(dir3741001); + const $table = $('table.table tr'); - const links = await frScraper.extractLinks($table, true); + const links = await frScraper.extractLinks($table, true); + const linkCount = links.length; - - const linkCount = links.length; - - t.equal(linkCount, 1, 'Scrapes the correct number of links (1)'); - t.deepEquals(links, dir3741_001Data, 'Links match the data'); - t.end(); + t.equal(linkCount, 1, 'Scrapes the correct number of links (1)'); + t.deepEquals(links, dir3741001Data, 'Links match the data'); + t.end(); }); t.test('FRANCE::Extract Details from Page 4/103', async t => { - const $ = cheerio.load(dir3741_002); - const $table = $('table.table tr'); + const $ = cheerio.load(dir3741002); + const $table = $('table.table tr'); - const links = await frScraper.extractLinks($table, true); + const links = await frScraper.extractLinks($table, true); + const linkCount = links.length; - - const linkCount = links.length; - - t.equal(linkCount, 3, 'Scrapes the correct number of links (3)'); - t.deepEquals(links, dir3741_002Data, 'Links match the data'); - t.end(); - }); + t.equal(linkCount, 3, 'Scrapes the correct number of links (3)'); + t.deepEquals(links, dir3741002Data, 'Links match the data'); + t.end(); + }); t.test('FRANCE::Extract Details from Page 11/103', async t => { - const $ = cheerio.load(dir3741_003); - const $table = $('table.table tr'); + const $ = cheerio.load(dir3741003); + const $table = $('table.table tr'); - const links = await frScraper.extractLinks($table, true); + const links = await frScraper.extractLinks($table, true); + const linkCount = links.length; - - - const linkCount = links.length; - - t.equal(linkCount, 2, 'Scrapes the correct number of links (2)'); - t.deepEquals(links, dir3741_003Data, 'Links match the data'); - t.end(); - }); + t.equal(linkCount, 2, 'Scrapes the correct number of links (2)'); + t.deepEquals(links, dir3741003Data, 'Links match the data'); + t.end(); + }); t.end(); }); diff --git a/tests/scrape.nl.js b/tests/scrape.nl.js index 793e298..45250f9 100644 --- a/tests/scrape.nl.js +++ b/tests/scrape.nl.js @@ -315,3 +315,19 @@ test('NL:: Scrape a Credit Service', async t => { t.end(); }); + +test('NL:: DIN-329 defects', async t => { + const defect2AHTML = fs.readFileSync('tests/data/nl/din329_d2_01.html'); + + t.test('NL::Defect 2', async t => { + const expectedJSON = jsonfile.readFileSync('tests/data/nl/din329_d2_01.json'); + + const output = await nlScraper.extractDetail(defect2AHTML); + + t.deepEquals(output, expectedJSON, 'Extracted Details from Page correctly'); + + t.end(); + }); + + t.end(); +}); diff --git a/tests/scrape.no.js b/tests/scrape.no.js index 99c0a34..a9a5f7d 100644 --- a/tests/scrape.no.js +++ b/tests/scrape.no.js @@ -225,6 +225,77 @@ test.test('Entity', async t => { t.end(); }); + test.test('NO:: Cross-Border format', async t => { + + t.test('NO:: Extract Cross-Border services 001 (PS)', async t => { + const crossborderHtml = fs.readFileSync('tests/data/no/cb_001_ps.html').toString(); + const expectedJSON = jsonfile.readFileSync('tests/data/no/cb_001_ps.json'); + + const output = await noScraper.recurseCrossborderHtml(crossborderHtml); + + t.deepEquals(output, expectedJSON, 'Extracted cross-border services from html'); + + t.end(); + }); + + t.test('NO:: Extract Cross-Border services 002 (PS)', async t => { + const crossborderHtml = fs.readFileSync('tests/data/no/cb_002_ps.html').toString(); + const expectedJSON = jsonfile.readFileSync('tests/data/no/cb_002_ps.json'); + + const output = await noScraper.recurseCrossborderHtml(crossborderHtml); + + t.deepEquals(output, expectedJSON, 'Extracted cross-border services from html'); + + t.end(); + }); + + t.test('NO:: Extract Cross-Border services 003 (PS)', async t => { + const crossborderHtml = fs.readFileSync('tests/data/no/cb_003_ps.html').toString(); + const expectedJSON = jsonfile.readFileSync('tests/data/no/cb_003_ps.json'); + + const output = await noScraper.recurseCrossborderHtml(crossborderHtml); + + t.deepEquals(output, expectedJSON, 'Extracted cross-border services from html'); + + t.end(); + }); + + t.test('NO:: Extract Cross-Border services 004 (PS)', async t => { + const crossborderHtml = fs.readFileSync('tests/data/no/cb_004_ps.html').toString(); + const expectedJSON = jsonfile.readFileSync('tests/data/no/cb_004_ps.json'); + + const output = await noScraper.recurseCrossborderHtml(crossborderHtml); + + t.deepEquals(output, expectedJSON, 'Extracted cross-border services from html'); + + t.end(); + }); + + t.test('NO:: Extract Cross-Border services 005 (PS with an empty list item)', async t => { + const crossborderHtml = fs.readFileSync('tests/data/no/cb_005_ps_with_empty_item.html').toString(); + const expectedJSON = jsonfile.readFileSync('tests/data/no/cb_005_ps_with_empty_item.json'); + + const output = await noScraper.recurseCrossborderHtml(crossborderHtml); + + t.deepEquals(output, expectedJSON, 'Extracted cross-border services from html'); + + t.end(); + }); + + t.test('NO:: Extract Cross-Border services 006 (PS with multiple countries)', async t => { + const crossborderHtml = fs.readFileSync('tests/data/no/cb_006_em_with_multi_countries.html').toString(); + const expectedJSON = jsonfile.readFileSync('tests/data/no/cb_006_em_with_multi_countries.json'); + + const output = await noScraper.recurseCrossborderHtml(crossborderHtml); + + t.deepEquals(output, expectedJSON, 'Extracted cross-border services from html'); + + t.end(); + }); + + t.end(); + }); + t.end(); }); diff --git a/version.properties b/version.properties index 1f29be9..14ecfb0 100644 --- a/version.properties +++ b/version.properties @@ -1 +1 @@ -VERSION=1.0.1 +VERSION=1.0.2