final update
This commit is contained in:
Martin Donnelly
parent
105a2c0abe
commit
534fd67b5d
1
.gitignore
vendored
1
.gitignore
vendored
@ -159,3 +159,4 @@ artefacts/*
|
||||
|
||||
/archive.tar.gz
|
||||
/user/
|
||||
/zip
|
||||
|
||||
7
Jenkinsfile
vendored
7
Jenkinsfile
vendored
@ -1,6 +1,3 @@
|
||||
@Library('OpenBankingUK/ob-pipeline-library') _
|
||||
@Library('OpenBankingUK/devdir-jenkins-pipeline-lib') _
|
||||
|
||||
javaMsPipelinev2 {
|
||||
projectName='obdfcascrape'
|
||||
cluster='nca'
|
||||
}
|
||||
buildNodeMsPipeline {projectName='obdfcascrape'}
|
||||
|
||||
@ -25,14 +25,14 @@ function buildApps() {
|
||||
{ 'cron':'DE_CRON', 'start':'DE', 'name':'DE', 'script':'de.js', 'proxy': 'de', 'crontime': '0 0 * * *' }, // 03:55:38 - de free at 4:00
|
||||
{ 'cron':'NL_CRON', 'start':'NL', 'name':'NL', 'script':'nl.js', 'proxy': 'nl', 'crontime': '0 0 * * *' }, // 07:23:19 - nl free at 7:30
|
||||
{ 'cron':'PL_CRON', 'start':'PL', 'name':'PL', 'script':'pl.js', 'proxy': 'ch', 'crontime': '0 0 * * *' }, // 17:59:18 - ch free at 18:00
|
||||
{ 'cron':'DK_CRON', 'start':'DK', 'name':'DK', 'script':'dk.js', 'proxy': 'uk' },
|
||||
{ 'cron':'ES_CRON', 'start':'ES', 'name':'ES', 'script':'es.js', 'proxy': 'uk' },
|
||||
{ 'cron':'LV_CRON', 'start':'LV', 'name':'LV', 'script':'lv.js', 'proxy': 'nl', 'crontime': '30 7 * * *' }, // 13:56.232 - nl free at 7:45
|
||||
{ 'cron':'DK_CRON', 'start':'DK', 'name':'DK', 'script':'dk.js', 'proxy': 'de', 'crontime': '0 4 * * *' }, // 11:08.616 - de free at 4:15
|
||||
{ 'cron':'ES_CRON', 'start':'ES', 'name':'ES', 'script':'es.js', 'proxy': 'de', 'crontime': '15 4 * * *' }, // 36:44.523- de free at 4:55
|
||||
{ 'cron':'EE_CRON', 'start':'EE', 'name':'EE', 'script':'ee.js', 'proxy': 'de', 'crontime': '0 5 * * *' }, // 05:22:04.226 - de free after 10:30
|
||||
{ 'cron':'NO_CRON', 'start':'NO', 'name':'NO', 'script':'no.js', 'proxy': 'fr', 'crontime': '0 4 * * *' }, // 05:12:57.792 - fr free after 9:20
|
||||
{ 'cron':'GI_CRON', 'start':'GI', 'name':'GI', 'script':'gi.js', 'proxy': 'uk' },
|
||||
{ 'cron':'GR_CRON', 'start':'GR', 'name':'GR', 'script':'gr.js', 'proxy': 'uk' },
|
||||
{ 'cron':'MT_CRON', 'start':'MT', 'name':'MT', 'script':'mt.js', 'proxy': 'uk' },
|
||||
{ 'cron':'LV_CRON', 'start':'LV', 'name':'LV', 'script':'lv.js', 'proxy': 'uk' },
|
||||
{ 'cron':'NO_CRON', 'start':'NO', 'name':'NO', 'script':'no.js', 'proxy': 'uk' },
|
||||
{ 'cron':'EE_CRON', 'start':'EE', 'name':'EE', 'script':'ee.js', 'proxy': 'uk' },
|
||||
{ 'cron':'BG_CRON', 'start':'BG', 'name':'BG', 'script':'bg.js', 'proxy': 'uk' },
|
||||
{ 'cron':'AT_CRON', 'start':'AT', 'name':'AT', 'script':'at.js', 'proxy': 'uk' },
|
||||
{ 'cron':'FI_CRON', 'start':'FI', 'name':'FI', 'script':'fi.js', 'proxy': 'uk' },
|
||||
|
||||
@ -231,14 +231,17 @@ class Scraper extends EventEmitter {
|
||||
'--disable-gpu',
|
||||
'--window-size=1920x1080',
|
||||
'--hide-scrollbars',
|
||||
'--disable-default-apps',
|
||||
'--remote-debugging-port=9222'
|
||||
'--disable-default-apps'
|
||||
]
|
||||
}).catch((err) => {
|
||||
logger.error('Puppeteer failed to launch');
|
||||
logger.error(err);
|
||||
});
|
||||
|
||||
const browserVersion = await this.browser.version();
|
||||
|
||||
logger.info(`Browser version ${browserVersion}`);
|
||||
|
||||
this.browser.on('disconnected', () => {
|
||||
logger.warn('Browser has become detached!');
|
||||
|
||||
@ -309,9 +312,11 @@ class Scraper extends EventEmitter {
|
||||
|
||||
this.page.on('error', async err => {
|
||||
logger.warn('Page crashed', err);
|
||||
if (!this.detatchable) {
|
||||
await this._uploadError();
|
||||
logger.warn('page.onError::emit recover');
|
||||
this.emit('recover');
|
||||
}
|
||||
});
|
||||
|
||||
this.page.on('pageerror', async err => {
|
||||
@ -595,6 +600,7 @@ class Scraper extends EventEmitter {
|
||||
*
|
||||
* @param destPath
|
||||
* @param filename
|
||||
* @param glob
|
||||
* @returns {Promise<*>}
|
||||
* @private
|
||||
*/
|
||||
@ -1099,6 +1105,9 @@ class Scraper extends EventEmitter {
|
||||
async _done() {
|
||||
logger.info('<=- DONE -=>');
|
||||
|
||||
// OK To close the browser window now
|
||||
this.canDetach();
|
||||
|
||||
const now = new Date();
|
||||
|
||||
this.perf.finished = now.getTime();
|
||||
@ -1114,9 +1123,6 @@ class Scraper extends EventEmitter {
|
||||
|
||||
await this._archive();
|
||||
|
||||
// OK To close the browser window now
|
||||
this.canDetach();
|
||||
|
||||
await this._forcePageClose();
|
||||
|
||||
await this._killRunningBrowser();
|
||||
@ -1512,6 +1518,18 @@ class Scraper extends EventEmitter {
|
||||
await jsonfile.writeFileSync(filePath, json);
|
||||
}
|
||||
|
||||
async _getLocalStorage( ) {
|
||||
return await this.page.evaluate(() => {
|
||||
const json = {};
|
||||
for (let i = 0; i < localStorage.length; i++) {
|
||||
const key = localStorage.key(i);
|
||||
json[key] = localStorage.getItem(key);
|
||||
}
|
||||
|
||||
return json;
|
||||
});
|
||||
}
|
||||
|
||||
_checkFileExistsSync(filePath) {
|
||||
try {
|
||||
fs.accessSync(filePath, fs.F_OK);
|
||||
@ -1664,17 +1682,25 @@ class Scraper extends EventEmitter {
|
||||
async __recover(restartURL) {
|
||||
logger.warn(`*** RECONNECTING ${this.id} PAGE ***`);
|
||||
|
||||
let crashCount = 0;
|
||||
if (this.crashLog.has(this.lastUrl)) {
|
||||
let crashCount = this.crashLog.get(this.lastUrl);
|
||||
crashCount = this.crashLog.get(this.lastUrl);
|
||||
crashCount++;
|
||||
this.crashLog.set(this.lastUrl, crashCount);
|
||||
|
||||
if (crashCount >= 3)
|
||||
logger.error('The page has crashed more than 3 times', this.lastUrl);
|
||||
|
||||
if (crashCount >= 10) {
|
||||
logger.error('10 times on the same page is enough', this.lastUrl);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
else
|
||||
this.crashLog.set(this.lastUrl, 1);
|
||||
|
||||
if (crashCount < 10) {
|
||||
if (this.browserCrashed) await this._initBrowser(true);
|
||||
|
||||
await this._createBrowserPage();
|
||||
@ -1685,18 +1711,25 @@ class Scraper extends EventEmitter {
|
||||
this.processNewPage();
|
||||
});
|
||||
|
||||
const onHold = (crashCount >= 3) ? (90000 * crashCount) : 0;
|
||||
const antiCollision = 125 + (Math.floor(Math.random() * (15 - 1)) * 500);
|
||||
const timeout = 90000 + antiCollision;
|
||||
const timeout = 90000 + antiCollision + onHold;
|
||||
|
||||
logger.info(`🚨 Restarting in ${(timeout / 1000).toFixed(2)} seconds.`);
|
||||
|
||||
setTimeout(async() => {
|
||||
logger.warn('Attempting recovery..');
|
||||
logger.warn(`Attempting recovery to ${restartURL}`);
|
||||
|
||||
await this.restart(restartURL);
|
||||
}, timeout);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param restartURL
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async restart(restartURL) {
|
||||
const rURL = restartURL || this.lastUrl;
|
||||
logger.info(`Restarting ${this.id} // Going to ${rURL}`);
|
||||
@ -1704,6 +1737,12 @@ class Scraper extends EventEmitter {
|
||||
await this._goto(rURL);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param filename
|
||||
* @param data
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async saveFile(filename, data) {
|
||||
try{
|
||||
fs.writeFileSync(filename, data);
|
||||
@ -1713,6 +1752,40 @@ class Scraper extends EventEmitter {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param s
|
||||
* @returns {string}
|
||||
*/
|
||||
soundex(s) {
|
||||
const a = s.toLowerCase().split(''),
|
||||
|
||||
codes = {
|
||||
'a': '', 'e': '', 'i': '', 'o': '', 'u': '',
|
||||
'b': 1, 'f': 1, 'p': 1, 'v': 1,
|
||||
'c': 2, 'g': 2, 'j': 2, 'k': 2, 'q': 2, 's': 2, 'x': 2, 'z': 2,
|
||||
'd': 3, 't': 3,
|
||||
'l': 4,
|
||||
'm': 5, 'n': 5,
|
||||
'r': 6
|
||||
};
|
||||
|
||||
const f = a.shift();
|
||||
let r = '';
|
||||
|
||||
r = f +
|
||||
a
|
||||
.map((v, i, a) => {
|
||||
return codes[v];
|
||||
})
|
||||
.filter((v, i, a) => {
|
||||
return ((i === 0) ? v !== codes[f] : v !== a[i - 1]);
|
||||
})
|
||||
.join('');
|
||||
|
||||
return (`${r }000`).slice(0, 4).toUpperCase();
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
module.exports = Scraper;
|
||||
|
||||
22
ncas/at.js
22
ncas/at.js
@ -89,9 +89,9 @@ class ATScrape extends Scraper {
|
||||
details['permissions'] = [];
|
||||
const permissionsDiv = $('div.modal-body');
|
||||
$(permissionsDiv).find('h4').each((i, item) => {
|
||||
const code = this._cleanUp($(item).text());
|
||||
const description = this._cleanUp($(item).next().text());
|
||||
details['permissions'].push({ 'code': code, 'description': description });
|
||||
const heading = this._cleanUp($(item).text());
|
||||
const body = $(item).next().html().split('<br>').map(x => this._cleanUp(x)).filter(x => x != "");
|
||||
details['permissions'].push({ heading, body });
|
||||
});
|
||||
}
|
||||
|
||||
@ -164,21 +164,19 @@ class ATScrape extends Scraper {
|
||||
|
||||
const entities = $('div.company-details-wrap');
|
||||
|
||||
const href = await this.page.url();
|
||||
|
||||
entities.each(async (i, item) => {
|
||||
const noWhiteSpace = /\W/g;
|
||||
|
||||
const details = this.extractEntityDetails($(item).html());
|
||||
const id = this._makeFieldName(details.name);
|
||||
const entity = removeAccents.remove(id.trim());
|
||||
const filename = [this.modePrefix[this.mode], entity.replace(noWhiteSpace, '_')].join('');
|
||||
const filename = [this.modePrefix[this.mode], entity.replace(noWhiteSpace, '_'), '.json'].join('');
|
||||
const filePath = `${this.path}/${filename}`.substring(0, 240);
|
||||
jsonfile.writeFile(`${filePath}.json`, { details });
|
||||
jsonfile.writeFile(`${filePath}`, { details });
|
||||
|
||||
this.getCurrentMode().links.push({
|
||||
'id': id,
|
||||
'href': await this.page.url(),
|
||||
'filename': filename
|
||||
});
|
||||
this.getCurrentMode().links.push({ id, href, filename });
|
||||
});
|
||||
|
||||
logger.info(`${entities.length} ${this.modeNames[this.mode]} entities scraped.`);
|
||||
@ -213,7 +211,6 @@ class ATScrape extends Scraper {
|
||||
await this.entityResultsPageProcessor();
|
||||
else
|
||||
logger.error(`Page url not recognised: ${pageUrl.href}`);
|
||||
|
||||
}
|
||||
|
||||
getCurrentMode() {
|
||||
@ -234,12 +231,11 @@ class ATScrape extends Scraper {
|
||||
getNextUrl() {
|
||||
if (this.getCurrentMode().urlStep < this.getCurrentMode().urls.length - 1)
|
||||
this.getCurrentMode().urlStep++;
|
||||
else {
|
||||
else
|
||||
if (this.mode < this.modeNames.length - 1)
|
||||
this.mode++;
|
||||
else
|
||||
return null;
|
||||
}
|
||||
|
||||
return this.getCurrentMode().urls[this.getCurrentMode().urlStep];
|
||||
}
|
||||
|
||||
@ -268,9 +268,9 @@ class BEScrape extends Scraper {
|
||||
const id = this.getIdByEntityName(entity.name);
|
||||
|
||||
// create json file for each entity
|
||||
const filename = [this.modePrefix[this.mode], id].join('');
|
||||
const filename = [this.modePrefix[this.mode], id, '.json'].join('');
|
||||
const filePath = `${this.path}/${filename}`.substring(0, 240);
|
||||
jsonfile.writeFile(`${filePath}.json`, { 'details': entity , metadataFileName});
|
||||
jsonfile.writeFile(filePath, { 'details': entity , metadataFileName});
|
||||
|
||||
// add entity details to "links" so that index file can be generated later
|
||||
this.getCurrentMode().links.push({
|
||||
|
||||
@ -15,6 +15,8 @@ class CYScrape extends Scraper {
|
||||
super();
|
||||
this.setID('CY');
|
||||
|
||||
this.addToBlockFilters(['recaptcha']);
|
||||
|
||||
this.on('done', () => {
|
||||
this._done();
|
||||
});
|
||||
|
||||
31
ncas/cz.js
31
ncas/cz.js
@ -11,7 +11,7 @@ class CZScrape extends Scraper {
|
||||
|
||||
constructor() {
|
||||
super();
|
||||
this.id = 'CZ';
|
||||
this.setID('CZ');
|
||||
this.version = '0.0.1-3';
|
||||
|
||||
this.captchas = ['iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAj0lEQVRYR+2XQQ6AIAwE7f8fXeOBBIlll8bYSNZzhemUZMHc3Y/CzwSQMWBm06GtTDU1ghIAtGmkBNmgDZQBZDcejUQmoIEIYKb26Z/XANBMW+cjhABkQAZkQAZkYB8DLe0+i+PVeL3q+yhG8Q0vJBEA+5bZB6DvGN0TUde3tX75MGHnz9TRh5BZLFNTDnACDZUAsJw5oEAAAAAASUVORK5CYII=',
|
||||
@ -218,11 +218,11 @@ class CZScrape extends Scraper {
|
||||
*/
|
||||
async entityCompleter(serviceObject) {
|
||||
let cbFlag = false;
|
||||
|
||||
try{
|
||||
if (serviceObject.current.authLink !== '' && !serviceObject.current.authProcess) {
|
||||
await this._randomWait(this.page, 3, 5, 'Get Authorisations');
|
||||
|
||||
await this._goto(serviceObject.current.authLink, { 'waitUntil':'networkidle2' });
|
||||
await this._goto(serviceObject.current.authLink, { 'waitUntil':'networkidle0' });
|
||||
|
||||
return null;
|
||||
}
|
||||
@ -231,10 +231,14 @@ class CZScrape extends Scraper {
|
||||
await this._randomWait(this.page, 3, 5, 'Get CBs');
|
||||
// logger.info(`Crossborder for ${serviceObject.current.crossBorderLinks[serviceObject.current.crossBorderStep].name}`);
|
||||
|
||||
await this._goto(serviceObject.current.crossBorderLinks[serviceObject.current.crossBorderStep].href, { 'waitUntil':'networkidle2' });
|
||||
await this._goto(serviceObject.current.crossBorderLinks[serviceObject.current.crossBorderStep].href, { 'waitUntil':'networkidle0' });
|
||||
}
|
||||
else
|
||||
cbFlag = true;
|
||||
}
|
||||
catch( err) {
|
||||
logger.error(err);
|
||||
}
|
||||
|
||||
if( cbFlag === true) {
|
||||
const filename = serviceObject.links[serviceObject.step].fileName;
|
||||
@ -254,7 +258,7 @@ class CZScrape extends Scraper {
|
||||
if (serviceObject.step < serviceObject.items) {
|
||||
serviceObject.current = {};
|
||||
|
||||
await this._goto(serviceObject.links[serviceObject.step].href, { 'waitUntil':'networkidle2' });
|
||||
await this._goto(serviceObject.links[serviceObject.step].href, { 'waitUntil':'networkidle0' });
|
||||
}
|
||||
else
|
||||
this.emit('serviceDone');
|
||||
@ -635,7 +639,7 @@ class CZScrape extends Scraper {
|
||||
|
||||
await this._randomWait(this.page, 3, 5, 'First sub section');
|
||||
|
||||
await this._goto(serviceObject.sectionLinks[serviceObject.indexStep], { 'waitUntil':'networkidle2' });
|
||||
await this._goto(serviceObject.sectionLinks[serviceObject.indexStep], { 'waitUntil':'networkidle0' });
|
||||
}
|
||||
|
||||
/**
|
||||
@ -704,15 +708,16 @@ class CZScrape extends Scraper {
|
||||
if (serviceObject.indexStep >= serviceObject.sectionLinks.length) {
|
||||
this.inProgress = true;
|
||||
serviceObject.items = serviceObject.links.length;
|
||||
|
||||
await this._randomWait(this.page, 3, 5, 'First page');
|
||||
logger.info('goto', serviceObject.links[serviceObject.step].href);
|
||||
|
||||
await this._goto(serviceObject.links[serviceObject.step].href, { 'waitUntil':'networkidle2' });
|
||||
await this._goto(serviceObject.links[serviceObject.step].href, { 'waitUntil':'networkidle0' });
|
||||
}
|
||||
else {
|
||||
await this._randomWait(this.page, 3, 5, 'Next sub section');
|
||||
|
||||
await this._goto(serviceObject.sectionLinks[serviceObject.indexStep], { 'waitUntil':'networkidle2', 'timeout': 0 });
|
||||
await this._goto(serviceObject.sectionLinks[serviceObject.indexStep], { 'waitUntil':'networkidle0', 'timeout': 5000 });
|
||||
}
|
||||
}
|
||||
|
||||
@ -837,12 +842,13 @@ class CZScrape extends Scraper {
|
||||
*/
|
||||
async processNewPage() {
|
||||
// give the page a few seconds to settle
|
||||
const errorPages = ['https://apl.cnb.cz/apljerrsdad/undefined', 'chrome-error://chromewebdata/'];
|
||||
await this._randomWait(this.page, 3, 5);
|
||||
|
||||
const pageUrl = url.parse(await this.page.url());
|
||||
|
||||
if (pageUrl.href === 'chrome-error://chromewebdata/') {
|
||||
logger.warn('Directed to: chrome-error://chromewebdata/');
|
||||
if (errorPages.indexOf(pageUrl.href) !== -1) {
|
||||
logger.warn(`Directed to: ${pageUrl.href}`);
|
||||
this.emit('recover');
|
||||
|
||||
return;
|
||||
@ -880,7 +886,8 @@ class CZScrape extends Scraper {
|
||||
default:
|
||||
if (process.env.NODE_ENV) {
|
||||
await this._uploadError();
|
||||
throw new Error(`Unknown page: ${pageUrl}`);
|
||||
// throw new Error(`Unknown page: ${pageUrl}`);
|
||||
this.emit('recover');
|
||||
}
|
||||
else {
|
||||
logger.warn('processNewPage Fell through');
|
||||
@ -1044,7 +1051,7 @@ class CZScrape extends Scraper {
|
||||
//
|
||||
|
||||
await this.page.setViewport({ 'width': 1200, 'height': 800 });
|
||||
await this._goto(this.startPage, { 'waitUntil':'networkidle2' });
|
||||
await this._goto(this.startPage, { 'waitUntil':'networkidle0' });
|
||||
|
||||
await this._randomWait(this.page, 3, 5);
|
||||
}
|
||||
|
||||
40
ncas/de.js
40
ncas/de.js
@ -5,7 +5,7 @@ const cheerio = require('cheerio');
|
||||
const path = require('path');
|
||||
const jsonfile = require('jsonfile');
|
||||
const removeAccents = require('remove-accents-diacritics');
|
||||
const logger = require('log4js').getLogger('DE');
|
||||
const logger = require('log4js').getLogger('(DE)');
|
||||
const url = require('url');
|
||||
|
||||
logger.level = process.env.LOGGER_LEVEL || 'warn';
|
||||
@ -123,6 +123,7 @@ class DEScrape extends Scraper {
|
||||
}
|
||||
|
||||
async processCreditInstIndexPage() {
|
||||
try{
|
||||
const noWhiteSpace = /\W/g;
|
||||
logger.info('Building CI sub-index...');
|
||||
|
||||
@ -174,8 +175,14 @@ class DEScrape extends Scraper {
|
||||
this.emit('ciindexdone');
|
||||
}
|
||||
}
|
||||
catch( err) {
|
||||
logger.error(err);
|
||||
this.emit('recover');
|
||||
}
|
||||
}
|
||||
|
||||
async processCreditInstPage() {
|
||||
try{
|
||||
const noWhiteSpace = /\W/g;
|
||||
|
||||
const id = this.creditServices.links[this.creditServices.step].id;
|
||||
@ -212,6 +219,11 @@ class DEScrape extends Scraper {
|
||||
else
|
||||
this.emit('creditinstdone');
|
||||
}
|
||||
catch( err) {
|
||||
logger.error(err);
|
||||
this.emit('recover');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
@ -232,6 +244,7 @@ class DEScrape extends Scraper {
|
||||
* @returns {Promise<{description: T[] | jQuery, permissions: {original: Array, translated: Array}}>}
|
||||
*/
|
||||
async extractPaymentEntity(html) {
|
||||
try{
|
||||
const permissions = { 'original':[], 'translated':[] };
|
||||
|
||||
const newLine = /\n/g;
|
||||
@ -285,12 +298,18 @@ class DEScrape extends Scraper {
|
||||
|
||||
return { description, permissions };
|
||||
}
|
||||
catch( err) {
|
||||
logger.error(err);
|
||||
this.emit('recover');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async processEntity() {
|
||||
try{
|
||||
const noWhiteSpace = /\W/g;
|
||||
if (!this.subIndex.done) {
|
||||
// We should not be here quite yet, so add this to subindex;
|
||||
@ -345,6 +364,11 @@ class DEScrape extends Scraper {
|
||||
this.emit('processdone');
|
||||
}
|
||||
}
|
||||
catch( err) {
|
||||
logger.error(err);
|
||||
this.emit('reover');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
@ -453,12 +477,12 @@ class DEScrape extends Scraper {
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async attachEvents() {
|
||||
this.on('startcredit', async function() {
|
||||
this.on('startcredit', async () => {
|
||||
logger.info('Starting Credit Institutes');
|
||||
await this._goto(this.credit);
|
||||
});
|
||||
|
||||
this.on('processdone', async function() {
|
||||
this.on('processdone', async () => {
|
||||
logger.warn('Payment Entities done', this.paymentServices.items);
|
||||
|
||||
jsonfile.writeFileSync(`${this.path}/paymentServices.json`, { 'links': this.paymentServices.links });
|
||||
@ -469,19 +493,19 @@ class DEScrape extends Scraper {
|
||||
await this._goto(this.emoneyUrl);
|
||||
});
|
||||
|
||||
this.on('subindexdone', async function() {
|
||||
this.on('subindexdone', async () => {
|
||||
logger.info('Sub Index done', this.paymentServices.items);
|
||||
logger.info(this.paymentServices.links[this.paymentServices.step].href);
|
||||
await this._goto(this.paymentServices.links[this.paymentServices.step].href);
|
||||
});
|
||||
|
||||
this.on('indexdone', async function() {
|
||||
this.on('indexdone', async () => {
|
||||
logger.info('Index done', this.index.items);
|
||||
logger.info(this.index.links[this.index.step].href);
|
||||
await this._goto(this.index.links[this.index.step].href);
|
||||
});
|
||||
|
||||
this.on('ciindexdone', async function() {
|
||||
this.on('ciindexdone', async () => {
|
||||
logger.info('CI Index done', this.creditServices.items);
|
||||
logger.info(this.creditServices.links[this.creditServices.step].href);
|
||||
|
||||
@ -489,7 +513,7 @@ class DEScrape extends Scraper {
|
||||
await this._goto(newUrl);
|
||||
});
|
||||
|
||||
this.on('creditinstdone', async function() {
|
||||
this.on('creditinstdone', async () => {
|
||||
logger.debug('Credit Institutes done', this.paymentServices.items);
|
||||
|
||||
jsonfile.writeFileSync(`${this.path}/creditServices.json`, { 'links':this.creditServices.links });
|
||||
@ -499,7 +523,7 @@ class DEScrape extends Scraper {
|
||||
this.emit('done');
|
||||
});
|
||||
|
||||
this.on('nextsubindex', async function() {
|
||||
this.on('nextsubindex', async () => {
|
||||
logger.debug(this.index.links[this.index.step].href);
|
||||
await this._goto(this.index.links[this.index.step].href);
|
||||
});
|
||||
|
||||
89
ncas/dkV2.js
89
ncas/dkV2.js
@ -1,6 +1,6 @@
|
||||
const Scraper = require('../helpers/scraper');
|
||||
const path = require('path');
|
||||
const logger = require('log4js').getLogger('DK');
|
||||
const logger = require('log4js').getLogger('(DK)');
|
||||
const url = require('url');
|
||||
|
||||
logger.level = process.env.LOGGER_LEVEL || 'warn';
|
||||
@ -9,7 +9,7 @@ class DKScrape extends Scraper {
|
||||
|
||||
constructor(checkForLock = true) {
|
||||
super();
|
||||
this.id = 'DK';
|
||||
this.setID('DK');
|
||||
|
||||
this.on('done', () => {
|
||||
this._done();
|
||||
@ -55,6 +55,7 @@ class DKScrape extends Scraper {
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async renameFile() {
|
||||
if (!this.errored) {
|
||||
const filename = this.filenames[this.step];
|
||||
|
||||
const sourceFile = 'Finanstilsynets virksomhedsregister - SQL.xlsx';
|
||||
@ -63,6 +64,11 @@ class DKScrape extends Scraper {
|
||||
const newFile = `${this.path}/${filename}.xlsx`;
|
||||
|
||||
await this._renameFile(origFile, newFile);
|
||||
|
||||
await this._randomWait(this.page, 5, 7, 'after renameFile');
|
||||
}
|
||||
else
|
||||
logger.warn('Skipping renameFile');
|
||||
}
|
||||
|
||||
/**
|
||||
@ -70,17 +76,26 @@ class DKScrape extends Scraper {
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async clickReturn() {
|
||||
if (!this.errored) {
|
||||
logger.debug('clickReturn');
|
||||
await this.iframe.waitForSelector('#lsAnalysisPage > div > div:nth-child(2)', { 'visible':true, 'timeout':75000 }).then(async (elm) => {
|
||||
console.log('found');
|
||||
await elm.click({ 'delay':90 });
|
||||
}).catch((e) => {
|
||||
logger.error('iframe missing stuff', e);
|
||||
// pageLoaded = false;
|
||||
});
|
||||
|
||||
await this._randomWait(this.page, 2, 3, 'after clickReturn click');
|
||||
await this._randomWait(this.page, 5, 7, 'after clickReturn click');
|
||||
this.step++;
|
||||
}).catch((e) => {
|
||||
logger.error('iframe missing stuff (clickReturn)', e);
|
||||
// pageLoaded = false;
|
||||
this.emit('recover');
|
||||
});
|
||||
}
|
||||
else
|
||||
logger.warn('Skipping clickReturn');
|
||||
|
||||
/*
|
||||
await this._randomWait(this.page, 5, 7, 'after clickReturn click');
|
||||
this.step++;
|
||||
*/
|
||||
}
|
||||
|
||||
/**
|
||||
@ -88,6 +103,7 @@ class DKScrape extends Scraper {
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async scrollContainer() {
|
||||
if (!this.errored) {
|
||||
await this.page.evaluate(() => {
|
||||
console.log('window.innerWidth', window.innerWidth);
|
||||
window.scrollBy(window.innerWidth, window.innerHeight);
|
||||
@ -99,11 +115,14 @@ class DKScrape extends Scraper {
|
||||
await this._randomWait(this.page, 2, 3, 'bottom right scroll arrow');
|
||||
|
||||
for(let count = 0; count < 15; count++) {
|
||||
this.page.mouse.click(1061, 437, { 'delay':500 });
|
||||
await this._randomWait(this.page, 1, 2, 'scrolling');
|
||||
this.page.mouse.click(1061, 437, { 'delay':821 });
|
||||
await this._randomWait(this.page, 4, 7, 'scrolling');
|
||||
}
|
||||
|
||||
await this._randomWait(this.page, 4, 5, 'after scroll');
|
||||
await this._randomWait(this.page, 5, 7, 'after scroll');
|
||||
}
|
||||
else
|
||||
logger.warn('Skipping scrollContainer');
|
||||
}
|
||||
|
||||
/**
|
||||
@ -111,6 +130,7 @@ class DKScrape extends Scraper {
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async clickExport() {
|
||||
if (!this.errored) {
|
||||
logger.debug('clickExport');
|
||||
|
||||
await this.movePageToTop();
|
||||
@ -137,12 +157,16 @@ class DKScrape extends Scraper {
|
||||
await this.iframe.waitForSelector('div.lsDialogContent > div:nth-child(2)', { 'visible':true, 'timeout':75000 }).then(async (elm) => {
|
||||
console.log('found');
|
||||
await elm.click({ 'delay':90 });
|
||||
await this._randomWait(this.page, 5, 7, 'after clickExport click');
|
||||
}).catch((e) => {
|
||||
logger.error('iframe missing stuff', e);
|
||||
logger.error('iframe missing stuff (clickExport)', e);
|
||||
this.errored = true;
|
||||
this.emit('recover');
|
||||
// pageLoaded = false;
|
||||
});
|
||||
|
||||
await this._randomWait(this.page, 2, 3, 'after clickExport click');
|
||||
}
|
||||
else
|
||||
logger.warn('Skipping clickExport');
|
||||
}
|
||||
|
||||
/**
|
||||
@ -159,12 +183,13 @@ class DKScrape extends Scraper {
|
||||
await this.iframe.waitForSelector('#lsAnalysisPage > div > div:nth-child(11)', { 'visible':true, 'timeout':75000 }).then(async (elm) => {
|
||||
console.log('found');
|
||||
await elm.click({ 'delay':90 });
|
||||
await this._randomWait(this.page, 5, 7, 'after clickSearch click');
|
||||
}).catch((e) => {
|
||||
logger.error('iframe missing stuff', e);
|
||||
// pageLoaded = false;
|
||||
this.emit('recover');
|
||||
this.errored = true;
|
||||
});
|
||||
|
||||
await this._randomWait(this.page, 2, 3, 'after clickSearch click');
|
||||
}
|
||||
|
||||
/**
|
||||
@ -191,7 +216,7 @@ class DKScrape extends Scraper {
|
||||
|
||||
this.page.mouse.click(400, 434);
|
||||
|
||||
await this._randomWait(this.page, 2, 3, 'Click 400, 434');
|
||||
await this._randomWait(this.page, 5, 7, 'Click 400, 434');
|
||||
}
|
||||
|
||||
/**
|
||||
@ -219,7 +244,7 @@ class DKScrape extends Scraper {
|
||||
|
||||
this.page.mouse.click(400, 585);
|
||||
|
||||
await this._randomWait(this.page, 2, 3, 'Click 400, 585');
|
||||
await this._randomWait(this.page, 5, 7, 'Click 400, 585');
|
||||
}
|
||||
|
||||
/**
|
||||
@ -246,7 +271,7 @@ class DKScrape extends Scraper {
|
||||
|
||||
this.page.mouse.click(400, 473);
|
||||
|
||||
await this._randomWait(this.page, 2, 3, 'Click 400, 473');
|
||||
await this._randomWait(this.page, 5, 7, 'Click 400, 473');
|
||||
}
|
||||
|
||||
/**
|
||||
@ -273,7 +298,7 @@ class DKScrape extends Scraper {
|
||||
|
||||
this.page.mouse.click(400, 631);
|
||||
|
||||
await this._randomWait(this.page, 2, 3, 'Click 400, 631');
|
||||
await this._randomWait(this.page, 5, 7, 'Click 400, 631');
|
||||
}
|
||||
|
||||
/**
|
||||
@ -300,7 +325,7 @@ class DKScrape extends Scraper {
|
||||
|
||||
this.page.mouse.click(400, 473);
|
||||
|
||||
await this._randomWait(this.page, 2, 3, 'Click 400, 473');
|
||||
await this._randomWait(this.page, 5, 7, 'Click 400, 473');
|
||||
}
|
||||
|
||||
/**
|
||||
@ -393,7 +418,21 @@ class DKScrape extends Scraper {
|
||||
|
||||
}
|
||||
|
||||
while(!this.complete );
|
||||
while(!this.complete && !this.errored);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param restartURL
|
||||
* @returns {Promise<void>}
|
||||
* @private
|
||||
*/
|
||||
async __recover() {
|
||||
// need to reset the error status then restart
|
||||
logger.info('OVERLOAD __recover');
|
||||
this.errored = false;
|
||||
|
||||
super.__recover(this.startPage);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -410,7 +449,7 @@ class DKScrape extends Scraper {
|
||||
|
||||
await this.motions();
|
||||
}).catch((e) => {
|
||||
logger.error('processEntityDetails', e);
|
||||
logger.error('waitForIframe', e);
|
||||
// pageLoaded = false;
|
||||
});
|
||||
}
|
||||
@ -448,6 +487,8 @@ class DKScrape extends Scraper {
|
||||
this.step = 0;
|
||||
this.complete = false;
|
||||
|
||||
this.errored = false;
|
||||
|
||||
this.filenames = ['paymentServices01', 'paymentServices02', 'eMoney01', 'eMoney02', 'creditServices01', 'creditServices02'];
|
||||
|
||||
this.startPage = 'https://virksomhedsregister.finanstilsynet.dk/listeudtr%C3%A6k-en.html';
|
||||
@ -458,7 +499,7 @@ class DKScrape extends Scraper {
|
||||
logger.error(err);
|
||||
});
|
||||
|
||||
await this._initBrowser();
|
||||
await this._initBrowser(false);
|
||||
await this._createBrowserPage();
|
||||
|
||||
await this._makeResponsive();
|
||||
|
||||
@ -2,7 +2,7 @@ const Scraper = require('../helpers/scraper');
|
||||
const cheerio = require('cheerio');
|
||||
const path = require('path');
|
||||
const jsonfile = require('jsonfile');
|
||||
const logger = require('log4js').getLogger('EE');
|
||||
const logger = require('log4js').getLogger('(EE)');
|
||||
const url = require('url');
|
||||
const removeAccents = require('remove-accents-diacritics');
|
||||
|
||||
@ -12,7 +12,7 @@ class EEScrape extends Scraper {
|
||||
|
||||
constructor() {
|
||||
super();
|
||||
this.id = 'EE';
|
||||
this.setID('EE');
|
||||
|
||||
this.on('done', () => {
|
||||
this._done();
|
||||
@ -574,7 +574,7 @@ class EEScrape extends Scraper {
|
||||
|
||||
if (this.paymentServices.indexStep >= this.paymentServices.urls.length) {
|
||||
this.workMode = 1;
|
||||
logger.debug(JSON.stringify(this.paymentServices));
|
||||
// logger.debug(JSON.stringify(this.paymentServices));
|
||||
newUrl = this.paymentServices.links[this.paymentServices.step].href;
|
||||
}
|
||||
else
|
||||
|
||||
17
ncas/es.js
17
ncas/es.js
@ -1,7 +1,7 @@
|
||||
const Scraper = require('../helpers/scraper');
|
||||
const cheerio = require('cheerio');
|
||||
const path = require('path');
|
||||
const logger = require('log4js').getLogger('ES');
|
||||
const logger = require('log4js').getLogger('(ES)');
|
||||
const url = require('url');
|
||||
const querystring = require('querystring');
|
||||
const removeAccents = require('remove-accents-diacritics');
|
||||
@ -13,7 +13,7 @@ class ESScrape extends Scraper {
|
||||
|
||||
constructor() {
|
||||
super();
|
||||
this.id = 'ES';
|
||||
this.setID('ES');
|
||||
|
||||
this.on('done', () => {
|
||||
this._done();
|
||||
@ -279,7 +279,7 @@ class ESScrape extends Scraper {
|
||||
const { name, id } = serviceObject.links[serviceObject.step];
|
||||
logger.info(`Process ${this.modeTitles[this.mode]} entity ${serviceObject.step + 1} of ${serviceObject.items} // ${name}`);
|
||||
|
||||
await this.page.waitForSelector('td.tdContenido', { 'visible':true, 'timeout':7500 }); // Wait for buttons at bottom of table to be visible
|
||||
await this.page.waitForSelector('td.tdContenido', { 'visible':true, 'timeout':90000 }); // Wait for buttons at bottom of table to be visible
|
||||
|
||||
await this._randomWait(this.page, 3, 5);
|
||||
|
||||
@ -361,6 +361,7 @@ class ESScrape extends Scraper {
|
||||
await this.processRedirector();
|
||||
else {
|
||||
await this._uploadError();
|
||||
this.emit('recover');
|
||||
throw new Error(`Unknown page: ${pageUrl}`);
|
||||
}
|
||||
}
|
||||
@ -370,7 +371,7 @@ class ESScrape extends Scraper {
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async attachEvents() {
|
||||
this.on('serviceDone', async function() {
|
||||
this.on('serviceDone', async () => {
|
||||
switch (this.mode) {
|
||||
|
||||
case 0:
|
||||
@ -388,6 +389,8 @@ class ESScrape extends Scraper {
|
||||
}
|
||||
});
|
||||
|
||||
//
|
||||
|
||||
this.on('psindexdone', async () => {
|
||||
this.paymentServices.items = this.paymentServices.links.length;
|
||||
logger.info(`${this.paymentServices.items} paymentServices items indexed`);
|
||||
@ -397,6 +400,8 @@ class ESScrape extends Scraper {
|
||||
await this._goto(newUrl);
|
||||
});
|
||||
|
||||
//
|
||||
|
||||
this.on('emindexdone', async () => {
|
||||
this.emoneyServices.items = this.emoneyServices.links.length;
|
||||
logger.info(`${this.emoneyServices.items} emoneyServices items indexed`);
|
||||
@ -406,6 +411,8 @@ class ESScrape extends Scraper {
|
||||
await this._goto(newUrl);
|
||||
});
|
||||
|
||||
//
|
||||
|
||||
this.on('ciindexdone', async () => {
|
||||
this.creditServices.items = this.creditServices.links.length;
|
||||
logger.info(`${this.creditServices.items} creditServices items indexed`);
|
||||
@ -415,6 +422,8 @@ class ESScrape extends Scraper {
|
||||
await this._goto(newUrl);
|
||||
});
|
||||
|
||||
//
|
||||
|
||||
this.on('indexdone', async () => {
|
||||
switch (this.mode) {
|
||||
|
||||
|
||||
209
ncas/it.js
209
ncas/it.js
@ -2,7 +2,7 @@ const Scraper = require('../helpers/scraper');
|
||||
const cheerio = require('cheerio');
|
||||
const path = require('path');
|
||||
const jsonfile = require('jsonfile');
|
||||
const logger = require('log4js').getLogger('IT');
|
||||
const logger = require('log4js').getLogger('(IT)');
|
||||
const url = require('url');
|
||||
|
||||
logger.level = process.env.LOGGER_LEVEL || 'warn';
|
||||
@ -56,8 +56,12 @@ class ITscrape extends Scraper {
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async forceEnglish() {
|
||||
await this._randomWait(this.page, 2, 2, 'Force English');
|
||||
const localStorage = await this._getLocalStorage();
|
||||
|
||||
const userLang = localStorage['userLang'].trim();
|
||||
|
||||
if (userLang !== 'en') {
|
||||
await this._randomWait(this.page, 2, 2, 'Force English');
|
||||
await this.page.waitForSelector('#bs-example-navbar-collapse-1 > ul > li.dropdown > a', { 'visible':true, 'timeout':7500 }).then(async (elm) => {
|
||||
await elm.click({ 'delay':Scraper.notARobot() });
|
||||
await this._randomWait(this.page, 2, 2);
|
||||
@ -67,6 +71,7 @@ class ITscrape extends Scraper {
|
||||
|
||||
await this._findAndClick('#bs-example-navbar-collapse-1 > ul > li.dropdown.open > ul > li:nth-child(2) > a');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
@ -74,6 +79,8 @@ class ITscrape extends Scraper {
|
||||
*/
|
||||
async handleFrontPage() {
|
||||
let pageReturned = false;
|
||||
this.processing = false;
|
||||
|
||||
await this._randomWait(this.page, 3, 5, 'handleFrontPage');
|
||||
|
||||
await this.page.waitFor('ul.linkgroup a', { 'visible':true }).then(async (elm) => {
|
||||
@ -86,16 +93,16 @@ class ITscrape extends Scraper {
|
||||
await this.page.waitFor('#my-container > div.container > div', { 'visible':true, 'timeout':7500 }).then(() => {
|
||||
pageReturned = true;
|
||||
}).catch(async () => {
|
||||
logger.info('We didnt transition back correctly, forcing another click..\n');
|
||||
logger.info('(handleFrontPage) We didnt transition back correctly, forcing another click..\n');
|
||||
});
|
||||
|
||||
if (!pageReturned) {
|
||||
await this.page.hover('ul.linkgroup a').catch((err) => {
|
||||
logger.debug(err.name);
|
||||
logger.debug(err.name, 'hover ul.linkgroup a');
|
||||
});
|
||||
|
||||
await this.page.focus('ul.linkgroup a').catch((err) => {
|
||||
logger.debug(err.name);
|
||||
logger.debug(err.name, 'focus ul.linkgroup a');
|
||||
});
|
||||
|
||||
await this.page.waitFor('ul.linkgroup a', { 'visible':true }).then(async (elm) => {
|
||||
@ -106,7 +113,7 @@ class ITscrape extends Scraper {
|
||||
}
|
||||
}
|
||||
|
||||
while(!pageReturned);
|
||||
while(!pageReturned && !this.processing);
|
||||
|
||||
// Supervisory registers and lists
|
||||
}
|
||||
@ -126,6 +133,11 @@ class ITscrape extends Scraper {
|
||||
logger.warn('Ajax loading shroud not removed after 90 seconds');
|
||||
});
|
||||
|
||||
await this.page.waitForSelector('div.loading', { 'visible':true, 'timeout':10000 }).catch((e) => {
|
||||
logger.warn('Ajax loading shroud still there...');
|
||||
});
|
||||
// body > app-root > div:nth-child(2) > div
|
||||
|
||||
await this.page.waitForSelector('ul.nav.navbar-nav.navbar-center li a', { 'visible':false, 'timeout':90000 }).then(async (elm) => {
|
||||
await elm.click({ 'delay':Scraper.notARobot() });
|
||||
await this._randomWait(this.page, 5, 8, 'await transition');
|
||||
@ -291,17 +303,18 @@ class ITscrape extends Scraper {
|
||||
await this.page.waitForSelector('div.loading', { 'visible':false, 'timeout':25000 });
|
||||
|
||||
let btnSuccess = false;
|
||||
|
||||
let breakCount = 0;
|
||||
do {
|
||||
await this.page.waitForSelector('button.btn.btn-success', { 'visible':true, 'timeout':2500 }).then(async (elm) => {
|
||||
await this.page.waitForSelector('button.btn.btn-success', { 'visible':true, 'timeout':45000 }).then(async (elm) => {
|
||||
await elm.click({ 'delay':Scraper.notARobot() });
|
||||
}).catch(() => {
|
||||
btnSuccess = true;
|
||||
});
|
||||
await this._randomWait(this.page, 1, 1, 'preparePSSearch btnSuccess');
|
||||
breakCount++;
|
||||
}
|
||||
|
||||
while(!btnSuccess);
|
||||
while(!btnSuccess && breakCount < 5);
|
||||
|
||||
this.page.waitFor('app-int-albi-grid-result').then(async () => {
|
||||
//
|
||||
@ -328,66 +341,30 @@ class ITscrape extends Scraper {
|
||||
|
||||
/**
|
||||
*
|
||||
* @returns {Promise<{registry, authority, registers}>}
|
||||
* @param filePath
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async processPSDetail() {
|
||||
let registry = {}, registers = {}, authority = {};
|
||||
async processPSDetailV2(filePath) {
|
||||
await this._randomWait(this.page, 1, 3, 'processPSDetailV2: Screenshots');
|
||||
|
||||
await this._randomWait(this.page, 3, 3, 'processPSDetail: AJAX');
|
||||
|
||||
// await this._makeScreenshotV2(this.page, `${filePath}_main`, null);
|
||||
|
||||
await this.page.waitFor('#sub-navbar > app-int-albi > app-int-albi-details > div > div.card.card-title > span > span', { 'visible': true }).catch((err) => {
|
||||
logger.warn('AJAX data has failed to load');
|
||||
logger.debug(err);
|
||||
|
||||
return { registry, registers, authority };
|
||||
});
|
||||
|
||||
await this.page.waitFor('app-int-albi-details').then(async () => {
|
||||
await this.forceScrollToTop();
|
||||
|
||||
const body = await this.page.content();
|
||||
|
||||
registry = await this.extractPSRegistry(body);
|
||||
|
||||
await this._randomWait(this.page, 2, 2, 'processPSDetail app-int-albi-details');
|
||||
}).catch(async (err) => {
|
||||
if (process.env.NODE_ENV) {
|
||||
await this._uploadError();
|
||||
throw new Error('processPSDetail\n', err);
|
||||
}
|
||||
else
|
||||
logger.error('processPSDetail\n', err);
|
||||
});
|
||||
|
||||
await this._randomWait(this.page, 1, 1, 'processPSDetail after app-int-albi-details');
|
||||
|
||||
//
|
||||
// wait for the data
|
||||
|
||||
await this.forceScrollToTop();
|
||||
await this._makeScreenshotV2(this.page, `${filePath}_detail`, null);
|
||||
|
||||
// wait for Registers Tab
|
||||
await this.page.waitFor('#sub-navbar > app-int-albi > app-int-albi-details > div > div:nth-child(2) > ul > li:nth-child(2) > a', { 'visible': true, 'timeout':10000 }).then(async (elm) => {
|
||||
logger.debug('** Showing Registers Tab');
|
||||
await this.page.waitFor('#sub-navbar > app-int-albi > app-int-albi-details > div > div:nth-child(2) > ul > li:nth-child(2) > a',
|
||||
{ 'visible': true, 'timeout':10000 }).then(async (elm) => {
|
||||
await elm.click({ 'delay':90 });
|
||||
|
||||
await this._randomWait(this.page, 1, 2, 'processPSDetailV2 wait for Tab trans');
|
||||
|
||||
await this.page.waitFor('app-details-albi', { 'visible': true, 'timeout':10000 }).then(async () => {
|
||||
const body = await this.page.content();
|
||||
|
||||
registers = await this.extractPSRegisters(body);
|
||||
|
||||
await this._randomWait(this.page, 2, 2, 'processPSDetail app-details-albi');
|
||||
}).catch(async (err) => {
|
||||
if (process.env.NODE_ENV)
|
||||
// await this._uploadError();
|
||||
throw new Error('No tab transition\n', err);
|
||||
|
||||
else
|
||||
logger.error('No tab transition');
|
||||
await this._makeScreenshotV2(this.page, `${filePath}_reg`, null);
|
||||
await this._randomWait(this.page, 3, 5, 'processPSDetailV2 ss');
|
||||
}).catch(async () => {
|
||||
logger.info('No tab transition');
|
||||
});
|
||||
|
||||
await this._randomWait(this.page, 1, 1, 'processPSDetail after app-details-albi');
|
||||
}).catch((err) => {
|
||||
logger.warn('No "registers" Block...');
|
||||
logger.debug(err);
|
||||
@ -395,36 +372,29 @@ class ITscrape extends Scraper {
|
||||
|
||||
// wait for Activity Tab
|
||||
await this.forceScrollToTop();
|
||||
await this.page.waitFor('#sub-navbar > app-int-albi > app-int-albi-details > div > div:nth-child(2) > ul > li:nth-child(3) > a', { 'visible': true, 'timeout':10000 }).then(async (elm) => {
|
||||
logger.debug('** Showing Activity Tab');
|
||||
await this.page.waitFor('#sub-navbar > app-int-albi > app-int-albi-details > div > div:nth-child(2) > ul > li:nth-child(3) > a',
|
||||
{ 'visible': true, 'timeout':10000 }).then(async (elm) => {
|
||||
await elm.click({ 'delay':90 });
|
||||
let pageReturned = false;
|
||||
|
||||
await this._randomWait(this.page, 1, 2, 'processPSDetailV2 wait for Tab B trans');
|
||||
do
|
||||
await this.page.waitFor('app-details-att-autorizzate', { 'visible': true, 'timeout':10000 }).then(async () => {
|
||||
pageReturned = true;
|
||||
const body = await this.page.content();
|
||||
|
||||
authority = await this.extractPSAuthority(body);
|
||||
|
||||
await this._randomWait(this.page, 2, 2, 'processPSDetail app-details-att-autorizzate');
|
||||
}).catch(async (err) => {
|
||||
// screen shot
|
||||
await this._makeScreenshotV2(this.page, `${filePath}_activity`, null);
|
||||
await this._randomWait(this.page, 3, 5, 'processPSDetailV2 ss');
|
||||
}).catch(async () => {
|
||||
await this.forceScrollToTop();
|
||||
await this._findAndClick('#sub-navbar > app-int-albi > app-int-albi-details > div > div:nth-child(2) > ul > li:nth-child(3) > a');
|
||||
|
||||
if (process.env.NODE_ENV)
|
||||
throw new Error('No tab transition\n', err);
|
||||
|
||||
else
|
||||
logger.warn('No tab transition');
|
||||
logger.info('No tab transition');
|
||||
});
|
||||
while(!pageReturned);
|
||||
}).catch((err) => {
|
||||
logger.warn('No "Activity" Block...');
|
||||
logger.debug(err);
|
||||
});
|
||||
|
||||
return { registry, registers, authority };
|
||||
}
|
||||
|
||||
/**
|
||||
@ -443,8 +413,9 @@ class ITscrape extends Scraper {
|
||||
await this.page.waitFor('app-int-albi-grid-result').then(() => {
|
||||
pageReturned = true;
|
||||
}).catch(async (err) => {
|
||||
logger.warn('We didnt transition back correctly, forcing another click..\n', err);
|
||||
logger.warn('(returnToPSList) We didnt transition back correctly, forcing another click..\n', err);
|
||||
|
||||
await this._randomWait(this.page, 1, 1, 'one sec');
|
||||
await this.forceScrollToTop();
|
||||
|
||||
await this._findAndClick('#sub-navbar > giava-breadcrumb > ol > li:nth-child(3) > a');
|
||||
@ -482,7 +453,6 @@ class ITscrape extends Scraper {
|
||||
divs.each((index, item) => {
|
||||
const itemText = this._cleanUp($(item).text());
|
||||
const itemName = $(item).attr('col-id');
|
||||
// logger.info(`>> ${index}`, itemName, itemText);
|
||||
entries[itemName] = itemText;
|
||||
});
|
||||
|
||||
@ -521,6 +491,8 @@ class ITscrape extends Scraper {
|
||||
let maxPages = 0;
|
||||
let rowsInPass;
|
||||
|
||||
this.processing = true;
|
||||
|
||||
await this.psSetListCount(_defaultMaxPerPage);
|
||||
const maxRows = await this.psGetMaxRows();
|
||||
let remainingRows = maxRows;
|
||||
@ -594,32 +566,38 @@ class ITscrape extends Scraper {
|
||||
serviceObject.links.push({ uid, 'fileName':`${fileName}.json`, 'name':clickName });
|
||||
|
||||
// Go into the detail
|
||||
await clickable.click();
|
||||
await clickable.click().catch((e) => {
|
||||
logger.debug('something happend with the click', e);
|
||||
throw new Error(e);
|
||||
});
|
||||
|
||||
await this._randomWait(this.page, 3, 4, 'processAGTableV3 before next');
|
||||
|
||||
remainingRows--;
|
||||
|
||||
await this.page.waitFor('app-int-albi-details').then(
|
||||
|
||||
await this.doAlbiDetails(filePath, newEntry)
|
||||
|
||||
).catch(async (err) => {
|
||||
await this.page.waitFor('app-int-albi-details').catch(async (err) => {
|
||||
logger.error('No detail transition', err);
|
||||
this.emit('recover');
|
||||
|
||||
if (process.env.NODE_ENV)
|
||||
await this._uploadError();
|
||||
});
|
||||
|
||||
await this.doAlbiDetailsV2(filePath, newEntry).catch(async (err) => {
|
||||
logger.error('doAlbiDetailsV2 had an issue', err);
|
||||
this.emit('recover');
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (maxPages > 0) {
|
||||
if (maxPages > 0 && !this.warning) {
|
||||
logger.info('Clicking to the next page...');
|
||||
|
||||
const nextButton = await this.page.$$('#sub-navbar > app-int-albi > app-int-albi-grid-result > grid-pagination > div > div > div:nth-child(5) > button');
|
||||
const buttonDisabled = await this.page.evaluate(el => el.disabled, nextButton[0]);
|
||||
const buttonDisabled = await this.page.evaluate(el => el.disabled, nextButton[0]).catch((e) => {
|
||||
logger.warn('Could not mark the button as disabled');
|
||||
});
|
||||
if (!buttonDisabled) {
|
||||
this._findAndClick('#sub-navbar > app-int-albi > app-int-albi-grid-result > grid-pagination > div > div > div:nth-child(5) > button');
|
||||
await this._randomWait(this.page, 5, 5, 'processAGTableV3 next page click');
|
||||
@ -632,30 +610,64 @@ class ITscrape extends Scraper {
|
||||
this.emit('doneProcessingGrid');
|
||||
}
|
||||
|
||||
async doAlbiDetails(filePath, newEntry) {
|
||||
/**
|
||||
*
|
||||
* @param filePath
|
||||
* @param newEntry
|
||||
* @returns {Promise<string>}
|
||||
*/
|
||||
async doAlbiDetailsV2(filePath, newEntry) {
|
||||
const empty = { };
|
||||
|
||||
logger.debug('++ doAlbiDetailsV2');
|
||||
let data;
|
||||
try{
|
||||
// process the page
|
||||
const data = await this.processPSDetail();
|
||||
data.details = newEntry;
|
||||
|
||||
let breaker = 0;
|
||||
|
||||
do{
|
||||
await this._randomWait(this.page, 1, 3, 'Retrieve from localStorage');
|
||||
const localStorage = await this._getLocalStorage();
|
||||
|
||||
logger.debug('>> typeof(localStorage[\'intermediaryDetails\'])', typeof(localStorage['intermediaryDetails']));
|
||||
|
||||
if (typeof(localStorage['intermediaryDetails']) !== 'undefined' )
|
||||
data = JSON.parse(localStorage['intermediaryDetails']) ;
|
||||
else
|
||||
await this._randomWait(this.page, 2, 2, 'wait for the localStorage load');
|
||||
|
||||
logger.debug('>> typeof(data)', typeof(data));
|
||||
breaker++;
|
||||
}
|
||||
while((typeof(data) === 'undefined' || data === null) && breaker < 10);
|
||||
|
||||
if (typeof(data) === 'undefined' || JSON.stringify(empty) === JSON.stringify(data)) {
|
||||
logger.warn('localStorage.intermediaryDetails did not load');
|
||||
|
||||
throw new Error('localStorage.intermediaryDetails did not load');
|
||||
}
|
||||
else {
|
||||
await this.processPSDetailV2(filePath);
|
||||
|
||||
logger.info(`Saving ${filePath}.json`);
|
||||
await jsonfile.writeFile(`${filePath}.json`, data);
|
||||
|
||||
await this._randomWait(this.page, 5, 7, 'doAlbiDetails');
|
||||
await this._randomWait(this.page, 5, 7, 'doAlbiDetailsV2');
|
||||
|
||||
// Retun back to list
|
||||
|
||||
await this.returnToPSList();
|
||||
|
||||
await this._randomWait(this.page, 2, 2, 'doAlbiDetails after returnToPSList');
|
||||
await this._randomWait(this.page, 2, 2, 'doAlbiDetailsV2 after returnToPSList');
|
||||
}
|
||||
|
||||
// wArray.push([uid, clickName]);
|
||||
}
|
||||
catch (err) {
|
||||
logger.error('doAlbiDetails\n', err);
|
||||
logger.error('doAlbiDetailsV2\n', err);
|
||||
logger.debug('Converted data:', data);
|
||||
this.emit('recover');
|
||||
|
||||
if (process.env.NODE_ENV)
|
||||
await this._uploadError();
|
||||
}
|
||||
}
|
||||
|
||||
@ -719,9 +731,11 @@ class ITscrape extends Scraper {
|
||||
}, 2500));
|
||||
|
||||
this.on('recover', this._debounce(async () => {
|
||||
this.warning = true;
|
||||
clearTimeout(this.backOffTimer);
|
||||
|
||||
logger.warn('Backing off for 5 minutes..');
|
||||
this.processing = false;
|
||||
const timeout = (60 * 1000) * 5;
|
||||
|
||||
this.backOffTimer = setTimeout(() => {
|
||||
@ -731,6 +745,7 @@ class ITscrape extends Scraper {
|
||||
}, 30000));
|
||||
|
||||
this.on('restart', this._debounce(async() => {
|
||||
this.warning = false;
|
||||
clearTimeout(this.backOffTimer);
|
||||
|
||||
logger.warn('Restarting::');
|
||||
@ -804,7 +819,9 @@ class ITscrape extends Scraper {
|
||||
async start() {
|
||||
super._start();
|
||||
try {
|
||||
this.mode = 0;
|
||||
this.mode = 2;
|
||||
this.processing = false;
|
||||
this.warning = false;
|
||||
|
||||
this.modeTitles = ['Payment Service', 'EMoney', 'Credit Services'];
|
||||
|
||||
@ -846,7 +863,7 @@ class ITscrape extends Scraper {
|
||||
'workingIndex': 0
|
||||
};
|
||||
|
||||
this.startPage = this.paymentServices.urls[0];
|
||||
this.startPage = this.creditServices.urls[0];
|
||||
this.emoneyUrl = '';
|
||||
this.credit = '';
|
||||
|
||||
@ -858,7 +875,7 @@ class ITscrape extends Scraper {
|
||||
logger.warn(err);
|
||||
});
|
||||
|
||||
await this._initBrowser(true);
|
||||
await this._initBrowser();
|
||||
await this._createBrowserPage();
|
||||
|
||||
this.page.on('domcontentloaded', this._throttle(async () => {
|
||||
|
||||
@ -159,6 +159,7 @@ class LUScrape extends Scraper {
|
||||
}
|
||||
catch( err) {
|
||||
logger.error(err);
|
||||
this.emit('recover');
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
12
ncas/lv.js
12
ncas/lv.js
@ -2,7 +2,7 @@ const Scraper = require('../helpers/scraper');
|
||||
const cheerio = require('cheerio');
|
||||
const path = require('path');
|
||||
const jsonfile = require('jsonfile');
|
||||
const logger = require('log4js').getLogger('LV');
|
||||
const logger = require('log4js').getLogger('(LV)');
|
||||
const url = require('url');
|
||||
const removeAccents = require('remove-accents-diacritics');
|
||||
|
||||
@ -12,7 +12,7 @@ class LVScrape extends Scraper {
|
||||
|
||||
constructor() {
|
||||
super();
|
||||
this.id = 'LV';
|
||||
this.setID('LV');
|
||||
|
||||
this.on('done', () => {
|
||||
this._done();
|
||||
@ -57,7 +57,7 @@ class LVScrape extends Scraper {
|
||||
|
||||
const filename = this.modeNames[this.mode];
|
||||
|
||||
this._makeScreenshotV2(this.page, `${this.path}/${filename}_main_${serviceObject.indexStep}`, null);
|
||||
this._makeScreenshotV2(this.page, `${this.path}/${filename}_main_${serviceObject.indexStep}`, 1500);
|
||||
|
||||
this.emit('indexdone');
|
||||
}
|
||||
@ -264,7 +264,7 @@ class LVScrape extends Scraper {
|
||||
|
||||
await this._randomWait(this.page, 3, 5);
|
||||
|
||||
await this._makeScreenshotV2(this.page, `${filePath}_main`, null);
|
||||
await this._makeScreenshotV2(this.page, `${filePath}_main`, 2000);
|
||||
|
||||
const body = await this.page.content();
|
||||
|
||||
@ -278,7 +278,8 @@ class LVScrape extends Scraper {
|
||||
const qualifyHoldings = await this.extractEntitySubSections(body, 'h2:contains("Qualifying holdings")');
|
||||
|
||||
// --
|
||||
await jsonfile.writeFile(`${filePath}.json`, { details, marketSegments, relatedPersons, licenses, sanctions, qualifyHoldings });
|
||||
|
||||
await jsonfile.writeFile(`${filePath}.json`, { details, licenses, marketSegments, qualifyHoldings, relatedPersons, sanctions });
|
||||
|
||||
await this._randomWait(this.page, 3, 5);
|
||||
|
||||
@ -293,6 +294,7 @@ class LVScrape extends Scraper {
|
||||
if (!err.message.includes('net::ERR_ABORTED') )
|
||||
logger.error('grabLink', err);
|
||||
});
|
||||
await this._randomWait(this.page, 2, 3);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
128
ncas/nl.js
128
ncas/nl.js
@ -3,7 +3,7 @@ const cheerio = require('cheerio');
|
||||
const path = require('path');
|
||||
const jsonfile = require('jsonfile');
|
||||
const removeAccents = require('remove-accents-diacritics');
|
||||
const logger = require('log4js').getLogger('NL');
|
||||
const logger = require('log4js').getLogger('(NL)');
|
||||
const url = require('url');
|
||||
|
||||
logger.level = process.env.LOGGER_LEVEL || 'warn';
|
||||
@ -179,6 +179,7 @@ class NLScrape extends Scraper {
|
||||
const noWhiteSpace = /\W/g;
|
||||
const urlSections = ['WFTBI', 'WFTEG', 'WFTKF'];
|
||||
const id = serviceObject.links[serviceObject.step].id;
|
||||
const hash = serviceObject.links[serviceObject.step].hash;
|
||||
|
||||
logger.info(`Process V2 ${this.modeTitles[this.mode]} entity ${serviceObject.step + 1} of ${serviceObject.items} // ${id}`);
|
||||
|
||||
@ -188,7 +189,8 @@ class NLScrape extends Scraper {
|
||||
|
||||
const filename = this._makeFileName(entity);
|
||||
|
||||
const filePath = `${this.path}/${filename}`.substring(0, 240);
|
||||
// DIN-329 - Trim a bit harder and append the unique hash
|
||||
const filePath = `${`${this.path}/${filename}`.substring(0, 215) }_${hash}`;
|
||||
|
||||
await this.page.waitForSelector('#contentcolumn > div.interactive-tabs > ol > li:nth-child(2) > a', { 'visible':true, 'timeout':7500 }).then(async (elm) => {
|
||||
await elm.click({ 'delay':Scraper.notARobot() });
|
||||
@ -337,7 +339,7 @@ class NLScrape extends Scraper {
|
||||
const options = await this.page.$$('#ddfilter option');
|
||||
const selects = ['2:12(1) Carrying on the business of a bank', '2:13(1) Carrying on the business of a bank'];
|
||||
const wantedOption = [];
|
||||
wantedOption.push(selects[this.creditServices.step]);
|
||||
wantedOption.push(selects[this.creditServices.menuStep]);
|
||||
for (const item of options) {
|
||||
const text = await this.page.evaluate(el => el.innerText, item);
|
||||
const value = await this.page.evaluate(el => el.value, item);
|
||||
@ -355,6 +357,18 @@ class NLScrape extends Scraper {
|
||||
}
|
||||
}
|
||||
|
||||
getQSIdShortHash(inStr) {
|
||||
const extractor = /([^?=&]+)(=([^&]*))?/g;
|
||||
|
||||
const matched = inStr.match(extractor);
|
||||
|
||||
if (matched.length === 2)
|
||||
return matched[1].slice(3, 11).toUpperCase();
|
||||
|
||||
else
|
||||
return '';
|
||||
}
|
||||
|
||||
/**
|
||||
* Process WFTBI / Payment Services
|
||||
* @returns {Promise<void>}
|
||||
@ -391,15 +405,19 @@ class NLScrape extends Scraper {
|
||||
|
||||
statutoryName = removeAccents.remove(statutoryName.trim()).replace(nonWhiteSpace, '_');
|
||||
|
||||
tradeName = removeAccents.remove(tradeName.trim()).replace(nonWhiteSpace, '_');
|
||||
tradeName = removeAccents.remove(tradeName.trim()).replace(nonWhiteSpace, '_') || '';
|
||||
|
||||
const id = (statutoryName === tradeName) ? statutoryName : `${statutoryName}-${tradeName}`;
|
||||
const statutoryNameExists = this.paymentServices.statutoryNames.indexOf(statutoryName.toLowerCase());
|
||||
|
||||
const id = (statutoryNameExists === -1) ? statutoryName : `${statutoryName}-${tradeName}`;
|
||||
|
||||
let href = cheerio(children.eq(0)).find('a').attr('href');
|
||||
href = href.concat('&locale=en_GB');
|
||||
// this is the one we want.
|
||||
const hash = this.getQSIdShortHash(href);
|
||||
|
||||
this.paymentServices.links.push({ id, href });
|
||||
href = href.concat('&locale=en_GB');
|
||||
|
||||
this.paymentServices.links.push({ id, href, hash });
|
||||
if (statutoryNameExists === -1) this.paymentServices.statutoryNames.push(statutoryName.toLowerCase());
|
||||
});
|
||||
|
||||
const next = $('a.next').attr('href') || '';
|
||||
@ -447,16 +465,19 @@ class NLScrape extends Scraper {
|
||||
|
||||
statutoryName = removeAccents.remove(statutoryName.trim()).replace(nonWhiteSpace, '_');
|
||||
|
||||
tradeName = removeAccents.remove(tradeName.trim()).replace(nonWhiteSpace, '_');
|
||||
tradeName = removeAccents.remove(tradeName.trim()).replace(nonWhiteSpace, '_') || '';
|
||||
|
||||
// const id = `${statutoryName}-${tradeName}`;
|
||||
const id = (statutoryName === tradeName) ? statutoryName : `${statutoryName}-${tradeName}`;
|
||||
const statutoryNameExists = this.emoneyServices.statutoryNames.indexOf(statutoryName.toLowerCase());
|
||||
|
||||
const id = (statutoryNameExists === -1) ? statutoryName : `${statutoryName}-${tradeName}`;
|
||||
|
||||
let href = cheerio(children.eq(0)).find('a').attr('href');
|
||||
href = href.concat('&locale=en_GB');
|
||||
// this is the one we want.
|
||||
const hash = this.getQSIdShortHash(href);
|
||||
|
||||
this.emoneyServices.links.push({ id, href });
|
||||
href = href.concat('&locale=en_GB');
|
||||
|
||||
this.emoneyServices.links.push({ id, href, hash });
|
||||
if (statutoryNameExists === -1) this.emoneyServices.statutoryNames.push(statutoryName.toLowerCase());
|
||||
});
|
||||
|
||||
const next = $('a.next').attr('href') || '';
|
||||
@ -506,19 +527,19 @@ class NLScrape extends Scraper {
|
||||
|
||||
statutoryName = removeAccents.remove(statutoryName.trim()).replace(nonWhiteSpace, '_');
|
||||
|
||||
tradeName = removeAccents.remove(tradeName.trim()).replace(nonWhiteSpace, '_');
|
||||
tradeName = removeAccents.remove(tradeName.trim()).replace(nonWhiteSpace, '_') || '';
|
||||
|
||||
const id = (statutoryName === tradeName) ? statutoryName : `${statutoryName}-${tradeName}`;
|
||||
const statutoryNameExists = this.creditServices.statutoryNames.indexOf(statutoryName.toLowerCase());
|
||||
|
||||
// const id = `${statutoryName}-${tradeName}`;
|
||||
const id = (statutoryNameExists === -1) ? statutoryName : `${statutoryName}-${tradeName}`;
|
||||
|
||||
let href = cheerio(children.eq(0)).find('a').attr('href');
|
||||
const hash = this.getQSIdShortHash(href);
|
||||
|
||||
href = href.concat('&locale=en_GB');
|
||||
// this is the one we want.
|
||||
|
||||
logger.debug({ id, href });
|
||||
|
||||
this.creditServices.links.push({ id, href });
|
||||
this.creditServices.links.push({ id, href, hash });
|
||||
if (statutoryNameExists === -1) this.creditServices.statutoryNames.push(statutoryName.toLowerCase());
|
||||
});
|
||||
|
||||
const next = $('a.next').attr('href') || '';
|
||||
@ -526,8 +547,8 @@ class NLScrape extends Scraper {
|
||||
if (next !== '')
|
||||
this._findAndClick('a.next');
|
||||
else
|
||||
if (this.creditServices.step === 0) {
|
||||
this.creditServices.step = 1;
|
||||
if (this.creditServices.menuStep === 0) {
|
||||
this.creditServices.menuStep = 1;
|
||||
await this._goto(this.credit);
|
||||
}
|
||||
else
|
||||
@ -591,23 +612,31 @@ class NLScrape extends Scraper {
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async restart() {
|
||||
// v2
|
||||
logger.info(`Restarting ${this.modeTitles[this.mode]}`);
|
||||
|
||||
switch (this.mode) {
|
||||
|
||||
case 2:
|
||||
this.emit('startProcessingCreditServices');
|
||||
break;
|
||||
|
||||
case 1:
|
||||
this.emit('startProcessingEMoneyServices');
|
||||
break;
|
||||
|
||||
case 0:
|
||||
default:
|
||||
if (this.mode === 0)
|
||||
if (this.paymentServices.links.length > 0) {
|
||||
this.emit('startProcessingPaymentServices');
|
||||
break;
|
||||
}
|
||||
else {
|
||||
await this._goto(this.startPage);
|
||||
}
|
||||
|
||||
else if(this.mode === 1)
|
||||
if (this.emoneyServices.links.length > 0) {
|
||||
this.emit('startProcessingEMoneyServices');
|
||||
}
|
||||
else {
|
||||
await this._goto(this.emoneyUrl);
|
||||
}
|
||||
|
||||
else if(this.mode === 2)
|
||||
if (this.creditServices.links.length > 0) {
|
||||
this.emit('startProcessingCreditServices');
|
||||
}
|
||||
else {
|
||||
await this._goto(this.credit);
|
||||
}
|
||||
}
|
||||
|
||||
@ -626,12 +655,13 @@ class NLScrape extends Scraper {
|
||||
this.processNewPage();
|
||||
});
|
||||
|
||||
const timeout = 90000;
|
||||
const antiCollision = 125 + (Math.floor(Math.random() * (15 - 1)) * 500);
|
||||
const timeout = 90000 + antiCollision;
|
||||
|
||||
setTimeout(async() => {
|
||||
setTimeout(() => {
|
||||
logger.warn('Attempting recovery..');
|
||||
|
||||
await this.restart();
|
||||
this.restart();
|
||||
}, timeout);
|
||||
}
|
||||
|
||||
@ -726,29 +756,33 @@ class NLScrape extends Scraper {
|
||||
this.mode = 0;
|
||||
try {
|
||||
this.paymentServices = {
|
||||
'done' : false,
|
||||
'items': 0,
|
||||
'links': [],
|
||||
'statutoryNames': [],
|
||||
'step': 0,
|
||||
'visited': false,
|
||||
'done' : false
|
||||
'visited': false
|
||||
};
|
||||
|
||||
this.emoneyServices = {
|
||||
'done' : false,
|
||||
'items': 0,
|
||||
'links': [],
|
||||
'searchDone' : false,
|
||||
'statutoryNames': [],
|
||||
'step': 0,
|
||||
'visited': false,
|
||||
'done' : false,
|
||||
'searchDone' : false
|
||||
'visited': false
|
||||
};
|
||||
|
||||
this.creditServices = {
|
||||
'done' : false,
|
||||
'items': 0,
|
||||
'links': [],
|
||||
'menuStep' : 0,
|
||||
'searchDone' : false,
|
||||
'statutoryNames': [],
|
||||
'step': 0,
|
||||
'visited': false,
|
||||
'done' : false,
|
||||
'searchDone' : false
|
||||
'visited': false
|
||||
};
|
||||
|
||||
this.startPage = 'https://www.dnb.nl/en/supervision/public-register/WFTBI/index.jsp';
|
||||
|
||||
351
ncas/no.js
351
ncas/no.js
@ -4,7 +4,7 @@ const Scraper = require('../helpers/scraper');
|
||||
const cheerio = require('cheerio');
|
||||
const path = require('path');
|
||||
const jsonfile = require('jsonfile');
|
||||
const logger = require('log4js').getLogger('NO');
|
||||
const logger = require('log4js').getLogger('(NO)');
|
||||
const url = require('url');
|
||||
const removeAccents = require('remove-accents-diacritics');
|
||||
|
||||
@ -14,7 +14,15 @@ class NOScrape extends Scraper {
|
||||
|
||||
constructor() {
|
||||
super();
|
||||
this.id = 'NO';
|
||||
this.setID('NO');
|
||||
|
||||
// treat these elements as block boundaries when scraping crossborder permissions
|
||||
this.blockBoundaries = 'div, li';
|
||||
|
||||
// override these values from the base class
|
||||
this.modePrefix = ['ps_', 'ps__in_', 'em_', 'em__in_', 'ci_', 'ci__in_'];
|
||||
this.modeNames = ['paymentServices', 'paymentServicesIncoming', 'emoneyServices', 'emoneyServicesIncoming', 'creditServices', 'creditServicesIncoming'];
|
||||
this.modeTitles = ['Payment Service', 'Payment Service Incoming', 'EMoney', 'EMoney Incoming', 'Credit Services', 'Credit Services Incoming'];
|
||||
|
||||
this.on('done', () => {
|
||||
this._done();
|
||||
@ -31,6 +39,70 @@ class NOScrape extends Scraper {
|
||||
});
|
||||
}
|
||||
|
||||
_reduceWhiteSpace(text) {
|
||||
return text.replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param html
|
||||
* @param selector
|
||||
* @returns {Promise<void>}
|
||||
*
|
||||
* Finds elements in the `html` with the given `selector`, but returns only the uppermost matched elements,
|
||||
* and not those that are nested within other matched elements.
|
||||
*/
|
||||
getUppermostElementsBySelector(html, selector) {
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
return $(selector).filter(function () {
|
||||
return $(this).parents(selector).length === 0;
|
||||
});
|
||||
}
|
||||
|
||||
getTextNotInMatchingElements(html, selector) {
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
$(selector)
|
||||
.remove()
|
||||
.end();
|
||||
|
||||
return $.text();
|
||||
}
|
||||
|
||||
recurseDOM(html, selector, level = 0) {
|
||||
const currentLevel = level + 1;
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
const result = [];
|
||||
|
||||
const blocks = this.getUppermostElementsBySelector(html, selector);
|
||||
|
||||
for (let i = 0; i < blocks.length; i++) {
|
||||
const block = blocks[i];
|
||||
|
||||
const rawName = this.getTextNotInMatchingElements($(block).html(), selector);
|
||||
const name = this._reduceWhiteSpace(rawName);
|
||||
|
||||
const blockHtml = $(block).html();
|
||||
|
||||
const data = this.recurseDOM(blockHtml, selector, currentLevel);
|
||||
|
||||
if (data === null)
|
||||
result.push(name);
|
||||
else
|
||||
result.push({
|
||||
'name': name,
|
||||
'data': data
|
||||
});
|
||||
}
|
||||
|
||||
if (result.length > 0)
|
||||
return result;
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param html
|
||||
@ -184,6 +256,23 @@ class NOScrape extends Scraper {
|
||||
return newObj;
|
||||
}
|
||||
|
||||
recurseCrossborderHtml(html) {
|
||||
try {
|
||||
// Wrap html in an outer div to give the recursion loop a root starting point
|
||||
const wrappedHtml = `<div>${html}</div>`;
|
||||
|
||||
// recurse the dom, using the selectors we have defined as block boundaries
|
||||
const results = this.recurseDOM(wrappedHtml, this.blockBoundaries);
|
||||
|
||||
// return the first item from our list (i.e. the outer div we added at the top of this function)
|
||||
return results[0];
|
||||
}
|
||||
catch(err) {
|
||||
logger.error('Error during recursion of cross-border HTML');
|
||||
logger.error(err);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param html
|
||||
@ -191,40 +280,15 @@ class NOScrape extends Scraper {
|
||||
*/
|
||||
extractEntityDetailCrossBorder(html) {
|
||||
try {
|
||||
const newObj = { };
|
||||
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
const header = $('h3.license-unit-label:contains("Cross-border services/classes")');
|
||||
|
||||
const detailBox = $(header).parent();
|
||||
|
||||
const children = $(detailBox).children();
|
||||
const crossborder = this.recurseCrossborderHtml(detailBox.html());
|
||||
|
||||
let curLabel = '';
|
||||
children.each(async (i, item) => {
|
||||
const tagName = $(item).prop('tagName');
|
||||
|
||||
if (tagName === 'H3') {
|
||||
curLabel = this._makeFieldName($(item).text());
|
||||
if (!newObj.hasOwnProperty(curLabel))
|
||||
newObj[curLabel] = [];
|
||||
}
|
||||
|
||||
if (['SPAN', 'A', 'P'].indexOf(tagName) !== -1)
|
||||
newObj[curLabel].push(this._cleanUp($(item).text()));
|
||||
|
||||
if(tagName === 'DIV' || tagName === 'UL') {
|
||||
if (!newObj.hasOwnProperty('data'))
|
||||
newObj['data'] = [];
|
||||
|
||||
const cbData = this.extractCrossBorderDetailsV2($(item).html());
|
||||
|
||||
newObj['data'].push(cbData);
|
||||
}
|
||||
});
|
||||
|
||||
return newObj;
|
||||
return crossborder;
|
||||
}
|
||||
catch( err) {
|
||||
logger.error(err);
|
||||
@ -254,6 +318,17 @@ class NOScrape extends Scraper {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param serviceObject
|
||||
* @param elm
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async selectCountryOption(serviceObject, elm) {
|
||||
const countryOption = serviceObject.country;
|
||||
await this.page.select('select.search-filter[aria-label="Choose country"]', countryOption);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param html
|
||||
@ -322,7 +397,7 @@ class NOScrape extends Scraper {
|
||||
|
||||
await this._randomWait(this.page, 5, 7);
|
||||
|
||||
this._makeScreenshotV2(this.page, `${this.path}/${filename}_main_${serviceObject.indexStep}`, null);
|
||||
this._makeScreenshotV2(this.page, `${this.path}/${filename}_main_${serviceObject.indexStep}`, 2500);
|
||||
|
||||
this.emit('indexdone');
|
||||
}
|
||||
@ -346,6 +421,18 @@ class NOScrape extends Scraper {
|
||||
logger.warn('Waiting for data timeout');
|
||||
});
|
||||
|
||||
await this._randomWait(this.page, 3, 5);
|
||||
|
||||
await this.page.waitForSelector('select.search-filter[aria-label="Choose country"]', { 'visible':true, 'timeout':7500 }).then(async (elm) => {
|
||||
await this.selectCountryOption(serviceObject, elm);
|
||||
}).catch((e) => {
|
||||
logger.error(e);
|
||||
logger.warn('No country select');
|
||||
});
|
||||
|
||||
// Give the country selection some time to filter the results
|
||||
await this._randomWait(this.page, 3, 5);
|
||||
|
||||
await this.page.waitForSelector('#js-konsregList > div > div', { 'visible':true, 'timeout':7500 }).then(async (elm) => {
|
||||
await this.processIndex(serviceObject);
|
||||
}).catch((e) => {
|
||||
@ -375,7 +462,8 @@ class NOScrape extends Scraper {
|
||||
{ 'find':'h2:contains("Agency debt collection on behalf of others")', 'blockType':'Debt collection' },
|
||||
{ 'find':'h2:contains("E-money institution")', 'blockType':'E-money institution' },
|
||||
{ 'find':'h2:contains("Investment firm")', 'blockType':'h2:contains("Investment firm")' },
|
||||
{ 'find':'h2:contains("Intermediator of loans and guarantees")', 'blockType':'Intermediator of loans and guarantees' }
|
||||
{ 'find':'h2:contains("Intermediator of loans and guarantees")', 'blockType':'Intermediator of loans and guarantees' },
|
||||
{ 'find':'h2:contains("Payment service provider with a limited authorisat")', 'blockType':'Payment service provider with a limited authorisat' }
|
||||
|
||||
];
|
||||
|
||||
@ -421,7 +509,7 @@ class NOScrape extends Scraper {
|
||||
|
||||
await this._randomWait(this.page, 5, 7);
|
||||
|
||||
await this._makeScreenshotV2(this.page, `${filePath}_main`, null);
|
||||
await this._makeScreenshotV2(this.page, `${filePath}_main`, 2500);
|
||||
|
||||
const body = await this.page.content();
|
||||
|
||||
@ -460,13 +548,25 @@ class NOScrape extends Scraper {
|
||||
break;
|
||||
|
||||
case 1:
|
||||
await this.buildIndex(this.emoneyServices);
|
||||
await this.buildIndex(this.paymentServicesIncoming);
|
||||
break;
|
||||
|
||||
case 2:
|
||||
await this.buildIndex(this.emoneyServices);
|
||||
break;
|
||||
|
||||
case 3:
|
||||
await this.buildIndex(this.emoneyServicesIncoming);
|
||||
break;
|
||||
|
||||
case 4:
|
||||
await this.buildIndex(this.creditServices);
|
||||
break;
|
||||
|
||||
case 5:
|
||||
await this.buildIndex(this.creditServicesIncoming);
|
||||
break;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@ -482,13 +582,25 @@ class NOScrape extends Scraper {
|
||||
break;
|
||||
|
||||
case 1:
|
||||
await this.processEntityDetails(this.emoneyServices);
|
||||
await this.processEntityDetails(this.paymentServicesIncoming);
|
||||
break;
|
||||
|
||||
case 2:
|
||||
await this.processEntityDetails(this.emoneyServices);
|
||||
break;
|
||||
|
||||
case 3:
|
||||
await this.processEntityDetails(this.emoneyServicesIncoming);
|
||||
break;
|
||||
|
||||
case 4:
|
||||
await this.processEntityDetails(this.creditServices);
|
||||
break;
|
||||
|
||||
case 5:
|
||||
await this.processEntityDetails(this.creditServicesIncoming);
|
||||
break;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@ -553,13 +665,25 @@ class NOScrape extends Scraper {
|
||||
break;
|
||||
|
||||
case 1:
|
||||
this.emit('emoneyServicesDone');
|
||||
this.emit('paymentServicesIncomingDone');
|
||||
break;
|
||||
|
||||
case 2:
|
||||
this.emit('emoneyServicesDone');
|
||||
break;
|
||||
|
||||
case 3:
|
||||
this.emit('emoneyServicesIncomingDone');
|
||||
break;
|
||||
|
||||
case 4:
|
||||
this.emit('creditServicesDone');
|
||||
break;
|
||||
|
||||
case 5:
|
||||
this.emit('creditServicesIncomingDone');
|
||||
break;
|
||||
|
||||
}
|
||||
});
|
||||
|
||||
@ -578,6 +702,21 @@ class NOScrape extends Scraper {
|
||||
await this._goto(newUrl);
|
||||
});
|
||||
|
||||
this.on('psincindexdone', async () => {
|
||||
let newUrl;
|
||||
this.paymentServicesIncoming.items = this.paymentServicesIncoming.links.length;
|
||||
logger.info(`${this.paymentServicesIncoming.items} items indexed`);
|
||||
|
||||
this.paymentServicesIncoming.indexStep++;
|
||||
if (this.paymentServicesIncoming.indexStep >= this.paymentServicesIncoming.wanted.length)
|
||||
newUrl = this.paymentServicesIncoming.links[this.paymentServicesIncoming.step].href;
|
||||
|
||||
else
|
||||
newUrl = this.paymentServicesIncoming.urls[0];
|
||||
|
||||
await this._goto(newUrl);
|
||||
});
|
||||
|
||||
this.on('emindexdone', async () => {
|
||||
let newUrl;
|
||||
this.emoneyServices.items = this.emoneyServices.links.length;
|
||||
@ -592,6 +731,20 @@ class NOScrape extends Scraper {
|
||||
await this._goto(newUrl);
|
||||
});
|
||||
|
||||
this.on('emincindexdone', async () => {
|
||||
let newUrl;
|
||||
this.emoneyServicesIncoming.items = this.emoneyServicesIncoming.links.length;
|
||||
logger.info(`${this.emoneyServicesIncoming.items} items indexed`);
|
||||
|
||||
this.emoneyServicesIncoming.indexStep++;
|
||||
if (this.emoneyServicesIncoming.indexStep >= this.emoneyServicesIncoming.urls.length)
|
||||
newUrl = this.emoneyServicesIncoming.links[this.emoneyServicesIncoming.step].href;
|
||||
else
|
||||
newUrl = this.emoneyServicesIncoming.urls[this.emoneyServicesIncoming.indexStep];
|
||||
|
||||
await this._goto(newUrl);
|
||||
});
|
||||
|
||||
this.on('ciindexdone', async () => {
|
||||
let newUrl;
|
||||
this.creditServices.items = this.creditServices.links.length;
|
||||
@ -606,6 +759,20 @@ class NOScrape extends Scraper {
|
||||
await this._goto(newUrl);
|
||||
});
|
||||
|
||||
this.on('ciincindexdone', async () => {
|
||||
let newUrl;
|
||||
this.creditServicesIncoming.items = this.creditServicesIncoming.links.length;
|
||||
logger.info(`${this.creditServicesIncoming.items} items indexed`);
|
||||
|
||||
this.creditServicesIncoming.indexStep++;
|
||||
if (this.creditServicesIncoming.indexStep >= this.creditServicesIncoming.urls.length)
|
||||
newUrl = this.creditServicesIncoming.links[this.creditServicesIncoming.step].href;
|
||||
else
|
||||
newUrl = this.creditServicesIncoming.urls[this.creditServicesIncoming.indexStep];
|
||||
|
||||
await this._goto(newUrl);
|
||||
});
|
||||
|
||||
this.on('indexdone', async () => {
|
||||
switch (this.mode) {
|
||||
|
||||
@ -614,13 +781,25 @@ class NOScrape extends Scraper {
|
||||
break;
|
||||
|
||||
case 1:
|
||||
this.emit('emindexdone');
|
||||
this.emit('psincindexdone');
|
||||
break;
|
||||
|
||||
case 2:
|
||||
this.emit('emindexdone');
|
||||
break;
|
||||
|
||||
case 3:
|
||||
this.emit('emincindexdone');
|
||||
break;
|
||||
|
||||
case 4:
|
||||
this.emit('ciindexdone');
|
||||
break;
|
||||
|
||||
case 5:
|
||||
this.emit('ciincindexdone');
|
||||
break;
|
||||
|
||||
}
|
||||
});
|
||||
|
||||
@ -634,7 +813,24 @@ class NOScrape extends Scraper {
|
||||
this.mode++;
|
||||
this.inProgress = false;
|
||||
|
||||
await this._goto(this.emoneyServices.urls[0]);
|
||||
await this._goto(this.paymentServicesIncoming.urls[0]);
|
||||
}
|
||||
catch (e) {
|
||||
logger.error(e);
|
||||
}
|
||||
});
|
||||
|
||||
this.on('paymentServicesIncomingDone', async () => {
|
||||
logger.warn('paymentServicesIncomingDone');
|
||||
try{
|
||||
this.paymentServicesIncoming.done = true;
|
||||
jsonfile.writeFileSync(`${this.path}/paymentServicesIncoming.json`, { 'links': this.paymentServicesIncoming.links });
|
||||
jsonfile.writeFileSync(`${this.debugPath}/paymentServicesIncoming.json`, this.paymentServicesIncoming);
|
||||
|
||||
this.mode++;
|
||||
this.inProgress = false;
|
||||
|
||||
await this._goto(this.emoneyServicesIncoming.urls[0]);
|
||||
}
|
||||
catch (e) {
|
||||
logger.error(e);
|
||||
@ -650,6 +846,22 @@ class NOScrape extends Scraper {
|
||||
this.mode++;
|
||||
this.inProgress = false;
|
||||
|
||||
await this._goto(this.emoneyServicesIncoming.urls[0]);
|
||||
}
|
||||
catch (e) {
|
||||
logger.error(e);
|
||||
}
|
||||
});
|
||||
|
||||
this.on('emoneyServicesIncomingDone', async () => {
|
||||
logger.warn('emoneyServicesIncomingDone');
|
||||
try{
|
||||
this.emoneyServicesIncoming.done = true;
|
||||
jsonfile.writeFileSync(`${this.path}/emoneyServicesIncoming.json`, { 'links':this.emoneyServicesIncoming.links });
|
||||
jsonfile.writeFileSync(`${this.debugPath}/emoneyServicesIncoming.json`, this.emoneyServicesIncoming);
|
||||
this.mode++;
|
||||
this.inProgress = false;
|
||||
|
||||
await this._goto(this.creditServices.urls[0]);
|
||||
}
|
||||
catch (e) {
|
||||
@ -666,6 +878,22 @@ class NOScrape extends Scraper {
|
||||
this.mode++;
|
||||
this.inProgress = false;
|
||||
|
||||
await this._goto(this.creditServicesIncoming.urls[0]);
|
||||
}
|
||||
catch (e) {
|
||||
logger.error(e);
|
||||
}
|
||||
});
|
||||
|
||||
this.on('creditServicesIncomingDone', async () => {
|
||||
logger.warn('creditServicesIncomingDone');
|
||||
try{
|
||||
this.creditServicesIncoming.done = true;
|
||||
jsonfile.writeFileSync(`${this.path}/creditServicesIncoming.json`, { 'links':this.creditServicesIncoming.links });
|
||||
jsonfile.writeFileSync(`${this.debugPath}/creditServicesIncoming.json`, this.creditServicesIncoming);
|
||||
this.mode++;
|
||||
this.inProgress = false;
|
||||
|
||||
this.emit('done');
|
||||
}
|
||||
catch (e) {
|
||||
@ -694,6 +922,21 @@ class NOScrape extends Scraper {
|
||||
'done' : false,
|
||||
'urls': ['https://www.finanstilsynet.no/en/finanstilsynets-registry/'],
|
||||
'wanted' : ['Payment institution', 'Agent of payment institution (company)', 'Payment service provider with a limited authorisat'],
|
||||
'country' : 'NOR',
|
||||
'sections' : [],
|
||||
'sectionLinks' : []
|
||||
};
|
||||
|
||||
this.paymentServicesIncoming = {
|
||||
'items': 0,
|
||||
'links': [],
|
||||
'step': 0,
|
||||
'indexStep': 0,
|
||||
'visited': false,
|
||||
'done' : false,
|
||||
'urls': ['https://www.finanstilsynet.no/en/finanstilsynets-registry/'],
|
||||
'wanted' : ['Payment institution', 'Agent of payment institution (company)', 'Payment service provider with a limited authorisat'],
|
||||
'country' : 'NOTNOR',
|
||||
'sections' : [],
|
||||
'sectionLinks' : []
|
||||
};
|
||||
@ -707,6 +950,21 @@ class NOScrape extends Scraper {
|
||||
'done' : false,
|
||||
'urls': ['https://www.finanstilsynet.no/en/finanstilsynets-registry/'],
|
||||
'wanted' : ['E-money institution'],
|
||||
'country' : 'NOR',
|
||||
'sections' : [],
|
||||
'sectionLinks' : []
|
||||
};
|
||||
|
||||
this.emoneyServicesIncoming = {
|
||||
'items': 0,
|
||||
'links': [],
|
||||
'step': 0,
|
||||
'indexStep': 0,
|
||||
'visited': false,
|
||||
'done' : false,
|
||||
'urls': ['https://www.finanstilsynet.no/en/finanstilsynets-registry/'],
|
||||
'wanted' : ['E-money institution'],
|
||||
'country' : 'NOTNOR',
|
||||
'sections' : [],
|
||||
'sectionLinks' : []
|
||||
};
|
||||
@ -722,13 +980,28 @@ class NOScrape extends Scraper {
|
||||
'started': false,
|
||||
'urls': ['https://www.finanstilsynet.no/en/finanstilsynets-registry/'],
|
||||
'wanted' : ['Bank', 'Branch of foreign credit institution', 'Credit Institution', 'Savings bank foundation'],
|
||||
'country' : 'NOR',
|
||||
'sections' : [],
|
||||
'sectionLinks' : []
|
||||
};
|
||||
|
||||
this.creditServicesIncoming = {
|
||||
'items': 0,
|
||||
'links': [],
|
||||
'step': 0,
|
||||
'indexStep': 0,
|
||||
'visited': false,
|
||||
'done' : false,
|
||||
'searchDone' : false,
|
||||
'started': false,
|
||||
'urls': ['https://www.finanstilsynet.no/en/finanstilsynets-registry/'],
|
||||
'wanted' : ['Bank', 'Branch of foreign credit institution', 'Credit Institution', 'Savings bank foundation'],
|
||||
'country' : 'NOTNOR',
|
||||
'sections' : [],
|
||||
'sectionLinks' : []
|
||||
};
|
||||
|
||||
this.startPage = this.paymentServices.urls[0];
|
||||
this.emoneyUrl = this.emoneyServices.urls[0];
|
||||
this.credit = this.creditServices.urls[0];
|
||||
|
||||
this.setPath(path.resolve(`${__dirname }/../artefacts/NO/FS`));
|
||||
|
||||
|
||||
28
ncas/pl.js
28
ncas/pl.js
@ -704,7 +704,7 @@ class PLScrape extends Scraper {
|
||||
}
|
||||
catch( err) {
|
||||
logger.error(err);
|
||||
this.emit('recover');
|
||||
this.emit('backoff');
|
||||
}
|
||||
}
|
||||
|
||||
@ -926,7 +926,10 @@ class PLScrape extends Scraper {
|
||||
});
|
||||
|
||||
if (doIndex)
|
||||
await this.processEntityIndex(this.paymentServices);
|
||||
await this.processEntityIndex(this.paymentServices).catch(async (err) => {
|
||||
logger.error('processEntityIndex catch: ', err);
|
||||
this.emit('restart');
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
@ -1021,7 +1024,7 @@ class PLScrape extends Scraper {
|
||||
|
||||
if (pageUrl.href === 'chrome-error://chromewebdata/') {
|
||||
logger.warn('Directed to: chrome-error://chromewebdata/');
|
||||
this.emit('recover');
|
||||
this.emit('backoff');
|
||||
|
||||
return;
|
||||
}
|
||||
@ -1102,7 +1105,7 @@ class PLScrape extends Scraper {
|
||||
*
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async restart() {
|
||||
async _restart() {
|
||||
logger.warn(`Tryng to restart ${this.modeTitles[this.mode]}`);
|
||||
|
||||
if (this.mode === 0) {
|
||||
@ -1145,9 +1148,10 @@ class PLScrape extends Scraper {
|
||||
if (this.backOffStep > this.backOffLimit) this.backOffStep = this.backOffLimit;
|
||||
|
||||
logger.warn(`Backing off for ${this.backOffStep * 5} minutes..`);
|
||||
const timeout = (60 * 1000) * (this.backOffStep * 5);
|
||||
const timeout = 300000; // (this.backOffStep * 5) * 60000;
|
||||
|
||||
await this._uploadError();
|
||||
logger.warn('timeout', timeout);
|
||||
// await this._uploadError();
|
||||
|
||||
this.backOffTimer = setTimeout(() => {
|
||||
this.emit('restart');
|
||||
@ -1166,15 +1170,19 @@ class PLScrape extends Scraper {
|
||||
}, 2500));
|
||||
|
||||
this.on('stall', () => {
|
||||
this.backoff();
|
||||
this.emit('backoff');
|
||||
});
|
||||
|
||||
this.on('backoff', () => {
|
||||
this.on('backoff', this._debounce( () => {
|
||||
this.backoff();
|
||||
});
|
||||
}, 10000));
|
||||
|
||||
/* this.on('backoff', () => {
|
||||
this.backoff();
|
||||
});*/
|
||||
|
||||
this.on('restart', async () => {
|
||||
await this.restart();
|
||||
await this._restart();
|
||||
});
|
||||
|
||||
this.on('entityComplete', () => {
|
||||
|
||||
@ -264,13 +264,15 @@ class PTScrape extends Scraper {
|
||||
|
||||
const pageUrl = url.parse(await this.page.url());
|
||||
|
||||
if (pageUrl.href === 'chrome-error://chromewebdata/') {
|
||||
logger.warn('Directed to: chrome-error://chromewebdata/');
|
||||
if (pageUrl.href === 'chrome-error://chromewebdata/' || pageUrl.pathname === null) {
|
||||
logger.warn('Directed to: chrome-error://chromewebdata/ or null');
|
||||
this.emit('recover');
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
logger.debug('processNewPage', pageUrl.pathname);
|
||||
|
||||
const splitPath = pageUrl.pathname.match(pathSplitter);
|
||||
|
||||
const pathname = splitPath[0];
|
||||
|
||||
12
ncas/sk.js
12
ncas/sk.js
@ -168,6 +168,7 @@ class SKScrape extends Scraper {
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async processEntityIndex(serviceObject) {
|
||||
try{
|
||||
const fields = ['referenceNumber', 'businessName', 'address', 'start', 'end', 'reason'];
|
||||
|
||||
const mouseDownDuration = Scraper.notARobot();
|
||||
@ -226,6 +227,11 @@ class SKScrape extends Scraper {
|
||||
}
|
||||
}
|
||||
}
|
||||
catch( err) {
|
||||
logger.error(err);
|
||||
this.emit('recover');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
@ -386,6 +392,7 @@ class SKScrape extends Scraper {
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
async processEntityDetail(serviceObject) {
|
||||
try{
|
||||
// level0 sublicctrl sublicctrl1 odd
|
||||
// level0 sublicctrl sublicctrl1 odd sublicshow shown
|
||||
|
||||
@ -457,6 +464,11 @@ class SKScrape extends Scraper {
|
||||
|
||||
this.entityCompleter(serviceObject);
|
||||
}
|
||||
catch( err) {
|
||||
logger.error(err);
|
||||
this.emit('recover');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
|
||||
147
package-lock.json
generated
147
package-lock.json
generated
@ -353,7 +353,7 @@
|
||||
},
|
||||
"ansi-colors": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-1.1.0.tgz",
|
||||
"resolved": "http://registry.npmjs.org/ansi-colors/-/ansi-colors-1.1.0.tgz",
|
||||
"integrity": "sha512-SFKX67auSNoVR38N3L+nvsPjOE0bybKTYbkf5tRvushrAPQ9V75huw0ZxBkKVeRU9kqH3d6HA4xTckbwZ4ixmA==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
@ -1253,7 +1253,7 @@
|
||||
"dependencies": {
|
||||
"uuid": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz",
|
||||
"resolved": "http://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz",
|
||||
"integrity": "sha1-Z+LoY3lyFVMN/zGOW/nc6/1Hsho="
|
||||
}
|
||||
}
|
||||
@ -1690,7 +1690,7 @@
|
||||
},
|
||||
"duplexer": {
|
||||
"version": "0.1.1",
|
||||
"resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz",
|
||||
"resolved": "http://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz",
|
||||
"integrity": "sha1-rOb/gIwc5mtX0ev5eXessCM0z8E=",
|
||||
"dev": true
|
||||
},
|
||||
@ -1711,7 +1711,7 @@
|
||||
},
|
||||
"readable-stream": {
|
||||
"version": "1.1.14",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
|
||||
"resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
|
||||
"integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
@ -2816,12 +2816,6 @@
|
||||
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz",
|
||||
"integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w=="
|
||||
},
|
||||
"get-own-enumerable-property-symbols": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.0.tgz",
|
||||
"integrity": "sha512-CIJYJC4GGF06TakLg8z4GQKvDsx9EMspVxOYih7LerEL/WosUnFIww45CGfxfeKHqlg3twgUrYRT1O3WQqjGCg==",
|
||||
"dev": true
|
||||
},
|
||||
"get-ssl-certificate": {
|
||||
"version": "2.3.1",
|
||||
"resolved": "https://registry.npmjs.org/get-ssl-certificate/-/get-ssl-certificate-2.3.1.tgz",
|
||||
@ -2979,7 +2973,7 @@
|
||||
},
|
||||
"readable-stream": {
|
||||
"version": "1.0.34",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
|
||||
"resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
|
||||
"integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
@ -3103,7 +3097,7 @@
|
||||
},
|
||||
"lodash": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/lodash/-/lodash-1.0.2.tgz",
|
||||
"resolved": "http://registry.npmjs.org/lodash/-/lodash-1.0.2.tgz",
|
||||
"integrity": "sha1-j1dWDIO1n8JwvT1WG2kAQ0MOJVE=",
|
||||
"dev": true
|
||||
},
|
||||
@ -3156,7 +3150,7 @@
|
||||
},
|
||||
"got": {
|
||||
"version": "6.7.1",
|
||||
"resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz",
|
||||
"resolved": "http://registry.npmjs.org/got/-/got-6.7.1.tgz",
|
||||
"integrity": "sha1-JAzQV4WpoY5WHcG0S0HHY+8ejbA=",
|
||||
"requires": {
|
||||
"create-error-class": "^3.0.0",
|
||||
@ -3179,7 +3173,7 @@
|
||||
},
|
||||
"gulp": {
|
||||
"version": "3.9.1",
|
||||
"resolved": "https://registry.npmjs.org/gulp/-/gulp-3.9.1.tgz",
|
||||
"resolved": "http://registry.npmjs.org/gulp/-/gulp-3.9.1.tgz",
|
||||
"integrity": "sha1-VxzkWSjdQK9lFPxAEYZgFsE4RbQ=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
@ -3212,7 +3206,7 @@
|
||||
},
|
||||
"chalk": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
|
||||
"resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
|
||||
"integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
@ -3225,19 +3219,19 @@
|
||||
},
|
||||
"minimist": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
|
||||
"resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
|
||||
"integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=",
|
||||
"dev": true
|
||||
},
|
||||
"semver": {
|
||||
"version": "4.3.6",
|
||||
"resolved": "https://registry.npmjs.org/semver/-/semver-4.3.6.tgz",
|
||||
"resolved": "http://registry.npmjs.org/semver/-/semver-4.3.6.tgz",
|
||||
"integrity": "sha1-MAvG4OhjdPe6YQaLWx7NV/xlMto=",
|
||||
"dev": true
|
||||
},
|
||||
"strip-ansi": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
|
||||
"resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
|
||||
"integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
@ -3282,13 +3276,13 @@
|
||||
},
|
||||
"async": {
|
||||
"version": "0.9.2",
|
||||
"resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz",
|
||||
"resolved": "http://registry.npmjs.org/async/-/async-0.9.2.tgz",
|
||||
"integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=",
|
||||
"dev": true
|
||||
},
|
||||
"bl": {
|
||||
"version": "0.9.5",
|
||||
"resolved": "https://registry.npmjs.org/bl/-/bl-0.9.5.tgz",
|
||||
"resolved": "http://registry.npmjs.org/bl/-/bl-0.9.5.tgz",
|
||||
"integrity": "sha1-wGt5evCF6gC8Unr8jvzxHeIjIFQ=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
@ -3346,7 +3340,7 @@
|
||||
},
|
||||
"lodash": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash/-/lodash-3.2.0.tgz",
|
||||
"resolved": "http://registry.npmjs.org/lodash/-/lodash-3.2.0.tgz",
|
||||
"integrity": "sha1-S/UKMkP5rrC6xBpV09WZBnWkYvs=",
|
||||
"dev": true
|
||||
},
|
||||
@ -3361,7 +3355,7 @@
|
||||
},
|
||||
"readable-stream": {
|
||||
"version": "1.0.34",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
|
||||
"resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
|
||||
"integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
@ -3379,7 +3373,7 @@
|
||||
},
|
||||
"tar-stream": {
|
||||
"version": "1.1.5",
|
||||
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.1.5.tgz",
|
||||
"resolved": "http://registry.npmjs.org/tar-stream/-/tar-stream-1.1.5.tgz",
|
||||
"integrity": "sha1-vpIYwTDCACnhB7D5Z/sj3gV50Tw=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
@ -3461,29 +3455,6 @@
|
||||
"through2": "^2.0.1"
|
||||
}
|
||||
},
|
||||
"gulp-changed-in-place": {
|
||||
"version": "2.3.0",
|
||||
"resolved": "https://registry.npmjs.org/gulp-changed-in-place/-/gulp-changed-in-place-2.3.0.tgz",
|
||||
"integrity": "sha1-3kFQwnbYwAUkUcry8MfSvhfnaIg=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"through2": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"gulp-debug": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/gulp-debug/-/gulp-debug-4.0.0.tgz",
|
||||
"integrity": "sha512-cn/GhMD2nVZCVxAl5vWao4/dcoZ8wUJ8w3oqTvQaGDmC1vT7swNOEbhQTWJp+/otKePT64aENcqAQXDcdj5H1g==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"chalk": "^2.3.0",
|
||||
"fancy-log": "^1.3.2",
|
||||
"plur": "^3.0.0",
|
||||
"stringify-object": "^3.0.0",
|
||||
"through2": "^2.0.0",
|
||||
"tildify": "^1.1.2"
|
||||
}
|
||||
},
|
||||
"gulp-gzip": {
|
||||
"version": "1.4.2",
|
||||
"resolved": "https://registry.npmjs.org/gulp-gzip/-/gulp-gzip-1.4.2.tgz",
|
||||
@ -3651,7 +3622,7 @@
|
||||
},
|
||||
"chalk": {
|
||||
"version": "1.1.3",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
|
||||
"resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
|
||||
"integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
@ -3670,7 +3641,7 @@
|
||||
},
|
||||
"minimist": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
|
||||
"resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
|
||||
"integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=",
|
||||
"dev": true
|
||||
},
|
||||
@ -3682,7 +3653,7 @@
|
||||
},
|
||||
"strip-ansi": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
|
||||
"resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
|
||||
"integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
@ -3974,12 +3945,6 @@
|
||||
"resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz",
|
||||
"integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo="
|
||||
},
|
||||
"irregular-plurals": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/irregular-plurals/-/irregular-plurals-2.0.0.tgz",
|
||||
"integrity": "sha512-Y75zBYLkh0lJ9qxeHlMjQ7bSbyiSqNW/UOPWDmzC7cXskL1hekSITh1Oc6JV0XCWWZ9DE8VYSB71xocLk3gmGw==",
|
||||
"dev": true
|
||||
},
|
||||
"is-absolute": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/is-absolute/-/is-absolute-1.0.0.tgz",
|
||||
@ -4140,7 +4105,7 @@
|
||||
},
|
||||
"is-obj": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz",
|
||||
"resolved": "http://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz",
|
||||
"integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8="
|
||||
},
|
||||
"is-path-cwd": {
|
||||
@ -4192,12 +4157,6 @@
|
||||
"has": "^1.0.1"
|
||||
}
|
||||
},
|
||||
"is-regexp": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/is-regexp/-/is-regexp-1.0.0.tgz",
|
||||
"integrity": "sha1-/S2INUXEa6xaYz57mgnof6LLUGk=",
|
||||
"dev": true
|
||||
},
|
||||
"is-relative": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/is-relative/-/is-relative-1.0.0.tgz",
|
||||
@ -4767,11 +4726,6 @@
|
||||
"p-is-promise": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"memory": {
|
||||
"version": "0.0.3",
|
||||
"resolved": "https://registry.npmjs.org/memory/-/memory-0.0.3.tgz",
|
||||
"integrity": "sha1-zgCclqXIreLyz0psmmlCprX+JvU="
|
||||
},
|
||||
"meow": {
|
||||
"version": "3.7.0",
|
||||
"resolved": "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz",
|
||||
@ -5002,11 +4956,6 @@
|
||||
"lower-case": "^1.1.1"
|
||||
}
|
||||
},
|
||||
"node-free": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/node-free/-/node-free-1.0.0.tgz",
|
||||
"integrity": "sha1-0rgIX0fCNh7dGfX/OpdKnLpJiwc="
|
||||
},
|
||||
"node-int64": {
|
||||
"version": "0.3.3",
|
||||
"resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.3.3.tgz",
|
||||
@ -6567,7 +6516,7 @@
|
||||
},
|
||||
"pause-stream": {
|
||||
"version": "0.0.11",
|
||||
"resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz",
|
||||
"resolved": "http://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz",
|
||||
"integrity": "sha1-/lo0sMvOErWqaitAPuLnO2AvFEU=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
@ -6684,15 +6633,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"plur": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/plur/-/plur-3.1.1.tgz",
|
||||
"integrity": "sha512-t1Ax8KUvV3FFII8ltczPn2tJdjqbd1sIzu6t4JL7nQ3EyeL/lTrj5PWKb06ic5/6XYDr65rQ4uzQEGN70/6X5w==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"irregular-plurals": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"pm2": {
|
||||
"version": "3.5.0",
|
||||
"resolved": "https://registry.npmjs.org/pm2/-/pm2-3.5.0.tgz",
|
||||
@ -6834,7 +6774,7 @@
|
||||
},
|
||||
"pretty-hrtime": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz",
|
||||
"resolved": "http://registry.npmjs.org/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz",
|
||||
"integrity": "sha1-t+PqQkNaTJsnWdmeDyAesZWALuE=",
|
||||
"dev": true
|
||||
},
|
||||
@ -6929,20 +6869,20 @@
|
||||
"resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.14.0.tgz",
|
||||
"integrity": "sha512-SayS2wUX/8LF8Yo2Rkpc5nkAu4Jg3qu+OLTDSOZtisVQMB2Z5vjlY2TdPi/5CgZKiZroYIiyUN3sRX63El9iaw==",
|
||||
"requires": {
|
||||
"debug": "^4.1.0",
|
||||
"extract-zip": "^1.6.6",
|
||||
"https-proxy-agent": "^2.2.1",
|
||||
"debug": "^3.1.0",
|
||||
"extract-zip": "^1.6.5",
|
||||
"https-proxy-agent": "^2.1.0",
|
||||
"mime": "^2.0.3",
|
||||
"progress": "^2.0.1",
|
||||
"progress": "^2.0.0",
|
||||
"proxy-from-env": "^1.0.0",
|
||||
"rimraf": "^2.6.1",
|
||||
"ws": "^6.1.0"
|
||||
"ws": "^3.0.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"debug": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
|
||||
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
|
||||
"version": "3.2.6",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
|
||||
"integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
|
||||
"requires": {
|
||||
"ms": "^2.1.1"
|
||||
}
|
||||
@ -6957,7 +6897,9 @@
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz",
|
||||
"integrity": "sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA==",
|
||||
"requires": {
|
||||
"async-limiter": "~1.0.0"
|
||||
"async-limiter": "~1.0.0",
|
||||
"safe-buffer": "~5.1.0",
|
||||
"ultron": "~1.1.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -7669,7 +7611,7 @@
|
||||
"dependencies": {
|
||||
"ansi-regex": {
|
||||
"version": "0.2.1",
|
||||
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-0.2.1.tgz",
|
||||
"resolved": "http://registry.npmjs.org/ansi-regex/-/ansi-regex-0.2.1.tgz",
|
||||
"integrity": "sha1-DY6UaWej2BQ/k+JOKYUl/BsiNfk=",
|
||||
"dev": true
|
||||
},
|
||||
@ -7681,7 +7623,7 @@
|
||||
},
|
||||
"chalk": {
|
||||
"version": "0.5.1",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-0.5.1.tgz",
|
||||
"resolved": "http://registry.npmjs.org/chalk/-/chalk-0.5.1.tgz",
|
||||
"integrity": "sha1-Zjs6ZItotV0EaQ1JFnqoN4WPIXQ=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
@ -7709,7 +7651,7 @@
|
||||
},
|
||||
"strip-ansi": {
|
||||
"version": "0.3.0",
|
||||
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-0.3.0.tgz",
|
||||
"resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-0.3.0.tgz",
|
||||
"integrity": "sha1-JfSOoiynkYfzF0pNuHWTR7sSYiA=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
@ -7731,7 +7673,7 @@
|
||||
},
|
||||
"stream-combiner": {
|
||||
"version": "0.2.2",
|
||||
"resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz",
|
||||
"resolved": "http://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz",
|
||||
"integrity": "sha1-rsjLrBd7Vrb0+kec7YwZEs7lKFg=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
@ -7813,17 +7755,6 @@
|
||||
"safe-buffer": "~5.1.0"
|
||||
}
|
||||
},
|
||||
"stringify-object": {
|
||||
"version": "3.3.0",
|
||||
"resolved": "https://registry.npmjs.org/stringify-object/-/stringify-object-3.3.0.tgz",
|
||||
"integrity": "sha512-rHqiFh1elqCQ9WPLIC8I0Q/g/wj5J1eMkyoiD6eoQApWHP0FtlK7rqnhmabL5VUY9JQCcqwwvlOaSuutekgyrw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"get-own-enumerable-property-symbols": "^3.0.0",
|
||||
"is-obj": "^1.0.1",
|
||||
"is-regexp": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"strip-ansi": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
|
||||
@ -8579,7 +8510,7 @@
|
||||
},
|
||||
"readable-stream": {
|
||||
"version": "1.0.34",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
|
||||
"resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
|
||||
"integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "obdfcascrape",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"description": "Scraping system to extract data from European Bank and Financial Services",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "nyc tape tests/**/*.js",
|
||||
@ -15,7 +15,7 @@
|
||||
"malta": "node mt.js",
|
||||
"debuglogs": "node debuglogs.js"
|
||||
},
|
||||
"author": "",
|
||||
"author": "Martin Donnelly",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"archiver": "^2.1.1",
|
||||
@ -39,7 +39,7 @@
|
||||
"moment": "^2.24.0",
|
||||
"node-free": "^1.0.0",
|
||||
"pm2": "^3.5.0",
|
||||
"puppeteer": "^1.14.0",
|
||||
"puppeteer": "1.15.0",
|
||||
"remove-accents-diacritics": "^1.0.2",
|
||||
"request": "^2.88.0",
|
||||
"tld-extract": "^1.0.1",
|
||||
|
||||
22
tests/data/at/ent_001.html
Normal file
22
tests/data/at/ent_001.html
Normal file
@ -0,0 +1,22 @@
|
||||
<h3>
|
||||
<span class="sr-only">Intermarket Bank AG Wien</span>
|
||||
<a href="#" class="company-details-toggle" rel="nofollow" title="Show Details" role="presentation" aria-hidden="true">
|
||||
<span class="glyphicon indicator glyphicon-chevron-down" aria-hidden="true"></span>
|
||||
Intermarket Bank AG </a>
|
||||
</h3>
|
||||
|
||||
<div class="company-details">
|
||||
<ul>
|
||||
|
||||
<li><strong>Category:</strong> Banks - Banks licensed in Austria<br>Banks - Austrian Banks in the EEA (freedom to provide services)<br></li><li><strong>Address:</strong> Am Belvedere 1 | 1100 Wien | Austria</li><li><strong>Contact:</strong><ul><li><strong>Phone:</strong> +43 5010028900</li><li><strong>Email:</strong> <a href="mailto:Meldewesen@Intermarket.at">Meldewesen@Intermarket.at</a></li><li><strong>Web:</strong> <a href="http://www.intermarket.at" target="_blank" rel="noopener">www.intermarket.at</a></li></ul></li><li><strong>Legal Identifiers:</strong><ul><li><strong>Bank identification number:</strong> 73600</li><li><strong>Commercial register number:</strong> 94144y</li></ul></li>
|
||||
</ul>
|
||||
|
||||
<p class="toggle-document-description-wrap"><i class="fa fa-eye" aria-hidden="true"></i> <a href="#" data-toggle="modal" data-target="#document-description-5ca20b072a24b">Show licenses</a></p><div class="modal fade document-description" id="document-description-5ca20b072a24b" role="dialog" tabindex="-1" aria-labelledby="modal-label-5ca20b072a24b"><div class="modal-dialog modal-md" role="document"><div class="modal-content"><div class="modal-header"><h6 class="modal-title" id="modal-label-5ca20b072a24b"><span class="sr-only">Description for:</span> Licenses for Intermarket Bank AG - Wien</h6></div><div class="modal-body"><h4>§ 1 Abs. 1 Z 1 BWG</h4><p>Die Entgegennahme fremder Gelder zur Verwaltung oder als Einlage (Einlagengeschäft);
|
||||
ausschließlich des Interbankenmarktes.</p><h4>§ 1 Abs. 1 Z 3 BWG</h4><p>Der Abschluss von Geldkreditverträgen und die Gewährung von Gelddarlehen (Kreditgeschäft)</p><h4>§ 1 Abs. 1 Z 4 BWG</h4><p>Der Kauf von Schecks und Wechseln, insbesondere die Diskontierung von Wechseln (Diskontgeschäft)</p><h4>§ 1 Abs. 1 Z 7 BWG</h4><p>Der Handel auf eigene oder fremde Rechnung mit<br> a) ausländischen Zahlungsmitteln (Devisen- und Valutengeschäft);</p><h4>§ 1 Abs. 1 Z 8 BWG</h4><p>Die Übernahme von Bürgschaften, Garantien und sonstigen Haftungen für andere, sofern die übernommene Verpflichtung auf Geldleistungen lautet (Garantiegeschäft)</p><h4>§ 1 Abs. 1 Z 16 BWG</h4><p>Der Ankauf von Forderungen aus Warenlieferungen oder Dienstleistungen, die Übernahme des Risikos der Einbringlichkeit solcher Forderungen ¿ ausgenommen die Kreditversicherung - und im Zusammenhang damit der Einzug solcher Forderungen (Factoringgeschäft)</p></div><div class="modal-footer"><button type="button" class="btn btn-default" data-dismiss="modal">Close description</button></div></div></div></div>
|
||||
|
||||
<div class="print-view-button-wrap">
|
||||
<i class="fa fa-print" aria-hidden="true" role="presentation"></i>
|
||||
<a class="company-db-print" href="https://www.fma.gv.at/wp-content/plugins/dw-fma/print.php" data-print="eNqFVMtu20gQvO9XNHSSsY4t2vKLFoyVEUcJjFxcskAM-DYkm-RA5FCYGUobB_tcJwvkYiA_ESBA9GOpHpLx-gHkILDZ01NdVd2Uig_izy6exqNEOR6dqzgKieN45Lxl9qNzF0c4ntd0ydWaM7ZMkWQP49GdXkmE4yiaTPow1f5TH95oNhKeINu0xttP_du8BbpWfZlRNYc-Z_HonfFsa2WX7OlSmSXNF3J0CgTluWg6iGiChBw7ekXds9IpG8cZaUM9_CyxtH8xlPVJ09ejzJdMV1dzGudQmjU1-YZWtlnrjMmxXQPR7XQg0vQoHq3KxvBgyZ_TQzqaQPjB6VmnHpbk6h-JJvGov8K10pXEB3h5z1XGG3Zs_vqf0D3le182nDjtuw5QuNls9vSzOnRJdNrDnxwe_-qd2uA2Znc2jaZTOPVvcK63xnXOHePWrJxe_PhKEc0Tt4fHLX6XN4vZPvKz1cVrDWOML7hgY1RZM8GhGqOnhQiwdNda-sh2oyrfmoIayanK0ZU2lSqYxn1gCnZpub3P_c45_aFah7dK8_YLKJWUsaNgQ4KJsBGRnt3ebH918Zzg4SOC6IcTgAGS1o0JxJaWM-3XbL3d3qM5tSajDFoWvNnel1aoDrWZshWXqBlfh1sPRF9uP33a_lq1eUD7Oy05xUJJsxtOS8eV2cV6uYRdY8L3XCIcXmu3bIzX_IvGUEzj_ux3HE6ecnirgF-RMGENwdxNohsWfVwwvpFmtfbdFpPaQbGrtvcm0-gF-bcKHprCocZ3XFx4rbGfr4Kej6pq_aMpvszs9On2bP9L2HarI1Ivt99sAQyVA22XFsoqsaKbEFxccl7oQ08u-4QoxwVSwb1dFORcMAsubr8JblPXXCIWG7jKZZPCEooLMtmKtetRKgX2nsZDv98ZHB0_WzKzHOb8prFZGB6AYVwi3SjLBsucD8ngPcRDzQOH3Y72gx2y9B-008vGkVxcwJeSWG0K0bFk7aG2wmTso34_vktLREF5gLwelt1pKRcHupnpmm5bp6SwVMhmCrMdWt21xYsNxm9U6hvh8dgj_IX8BMl1G1c,">Print view <span class="sr-only">(Opens in new tab)</span></a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
42
tests/data/at/ent_001.json
Normal file
42
tests/data/at/ent_001.json
Normal file
@ -0,0 +1,42 @@
|
||||
{
|
||||
"name": "Intermarket Bank AG",
|
||||
"address": "Am Belvedere 1, 1100 Wien, Austria",
|
||||
"phone": "+43 5010028900",
|
||||
"email": "Meldewesen@Intermarket.at",
|
||||
"website": "www.intermarket.at",
|
||||
"bankIdentificationNumber": "73600",
|
||||
"commercialRegisterNumber": "94144y",
|
||||
"categories": [
|
||||
"Banks - Banks licensed in Austria",
|
||||
"Banks - Austrian Banks in the EEA (freedom to provide services)"
|
||||
],
|
||||
"permissions": [
|
||||
{
|
||||
"heading": "§ 1 Abs. 1 Z 1 BWG",
|
||||
"body": ["Die Entgegennahme fremder Gelder zur Verwaltung oder als Einlage (Einlagengeschäft); ausschließlich des Interbankenmarktes."]
|
||||
},
|
||||
{
|
||||
"heading": "§ 1 Abs. 1 Z 3 BWG",
|
||||
"body": ["Der Abschluss von Geldkreditverträgen und die Gewährung von Gelddarlehen (Kreditgeschäft)"]
|
||||
},
|
||||
{
|
||||
"heading": "§ 1 Abs. 1 Z 4 BWG",
|
||||
"body": ["Der Kauf von Schecks und Wechseln, insbesondere die Diskontierung von Wechseln (Diskontgeschäft)"]
|
||||
},
|
||||
{
|
||||
"heading": "§ 1 Abs. 1 Z 7 BWG",
|
||||
"body": [
|
||||
"Der Handel auf eigene oder fremde Rechnung mit",
|
||||
"a) ausländischen Zahlungsmitteln (Devisen- und Valutengeschäft);"
|
||||
]
|
||||
},
|
||||
{
|
||||
"heading": "§ 1 Abs. 1 Z 8 BWG",
|
||||
"body": ["Die Übernahme von Bürgschaften, Garantien und sonstigen Haftungen für andere, sofern die übernommene Verpflichtung auf Geldleistungen lautet (Garantiegeschäft)"]
|
||||
},
|
||||
{
|
||||
"heading": "§ 1 Abs. 1 Z 16 BWG",
|
||||
"body": ["Der Ankauf von Forderungen aus Warenlieferungen oder Dienstleistungen, die Übernahme des Risikos der Einbringlichkeit solcher Forderungen ¿ ausgenommen die Kreditversicherung - und im Zusammenhang damit der Einzug solcher Forderungen (Factoringgeschäft)"]
|
||||
}
|
||||
]
|
||||
}
|
||||
21
tests/data/at/ent_002.html
Normal file
21
tests/data/at/ent_002.html
Normal file
@ -0,0 +1,21 @@
|
||||
<h3>
|
||||
<span class="sr-only">Bitpanda Payments GmbH Wien</span>
|
||||
<a href="#" class="company-details-toggle" rel="nofollow" title="Show Details" role="presentation" aria-hidden="true">
|
||||
<span class="glyphicon indicator glyphicon-chevron-down" aria-hidden="true"></span>
|
||||
Bitpanda Payments GmbH </a>
|
||||
</h3>
|
||||
|
||||
<div class="company-details">
|
||||
<ul>
|
||||
|
||||
<li><strong>Category:</strong> Payment institutions - Payment Institutions licensed in Austria<br></li><li><strong>Address:</strong> Jakov-Lind-Straße 2 | 1020 Wien | Austria</li>
|
||||
</ul>
|
||||
|
||||
<p class="toggle-document-description-wrap"><i class="fa fa-eye" aria-hidden="true"></i> <a href="#" data-toggle="modal" data-target="#document-description-5cc84f1a56622">Show licenses</a></p><div class="modal fade document-description" id="document-description-5cc84f1a56622" role="dialog" tabindex="-1" aria-labelledby="modal-label-5cc84f1a56622"><div class="modal-dialog modal-md" role="document"><div class="modal-content"><div class="modal-header"><h6 class="modal-title" id="modal-label-5cc84f1a56622"><span class="sr-only">Description for:</span> Licenses for Bitpanda Payments GmbH - Wien</h6></div><div class="modal-body"><h4>§ 1 Abs. 2 Z 3 ZaDiG 2018 - Zahlungsgeschäft</h4><p>Ausführung von Zahlungsvorgängen einschließlich des Transfers von Geldbeträgen auf ein Zahlungskonto beim Zahlungsdienstleister des Zahlungsdienstnutzers oder bei einem anderen Zahlungsdienstleister (Zahlungsgeschäft) a) Ausführung von Lastschriften einschließlich einmaliger Lastschriften (Lastschriftgeschäft); b) Ausführung von Zahlungsvorgängen mittels einer Zahlungskarte oder eines ähnlichen Instruments (Zahlungskartengeschäft); c) Ausführung von Überweisungen einschließlich Daueraufträgen (Überweisungsgeschäft); Die Konzession ist jedoch dahingehend eingeschränkt, dass die angebotenen Zahlungsdienste lediglich von Kunden der Bitpanda GmbH (FN 423018k) in Anspruch genommen werden dürfen;</p><h4>§ 1 Abs. 2 Z 6 ZaDiG 2018 - Finanztransfergeschäft</h4><p>Dienste, bei denen ohne Einrichtung eines Zahlungskontos auf den Namen des Zahlers oder des Zahlungsempfängers ein Geldbetrag eines Zahlers nur zum Transfer eines entsprechenden Betrags an den Zahlungsempfänger oder an einen anderen, im Namen des Zahlungsempfängers handelnden Zahlungsdienstleister entgegengenommen wird oder bei denen der Geldbetrag im Namen des Zahlungsempfängers entgegengenommen und diesem verfügbar gemacht wird (Finanztransfergeschäft) Die Konzession ist jedoch dahingehend eingeschränkt, dass die angebotenen Zahlungsdienste lediglich von Kunden der Bitpanda GmbH (FN 423018k) in Anspruch genommen werden dürfen;</p><h4>§ 1 Abs. 2 Z 7 ZaDiG 2018 - Zahlungsauslösedienst</h4><p>Dienste, die auf Antrag des Zahlungsdienstnutzers einen Zahlungsauftrag in Bezug auf ein bei einem anderen Zahlungsdienstleister geführtes Zahlungskonto auslösen (Zahlungsauslösedienste) Die Konzession ist jedoch dahingehend eingeschränkt, dass die angebotenen Zahlungsdienste lediglich von Kunden der Bitpanda GmbH (FN 423018k) in Anspruch genommen werden dürfen;</p></div><div class="modal-footer"><button type="button" class="btn btn-default" data-dismiss="modal">Close description</button></div></div></div></div>
|
||||
|
||||
<div class="print-view-button-wrap">
|
||||
<i class="fa fa-print" aria-hidden="true" role="presentation"></i>
|
||||
<a class="company-db-print" href="https://www.fma.gv.at/wp-content/plugins/dw-fma/print.php" data-print="eNrdVcFq20AQ_ZXBXCcbksZWQprIxpDgJm0TQqGFQm4rayxtLY3M7sohLv2TQi6G_kFPOVU_1tmVLNuRTXttb9qdmTdvZt6shO_5X7V_4rcCobHVF_6ZPZ_6LW0Uomn1te91_dZ7Mc3mh7eSwsOPRoniCcGztmO_tZAz-8UQva7XrT7H0jxWn58lkv18zbdZTkY9VqeLnHNIUbmRSNFl8_zWpTQzQaGAD-IxRTIartPgrbWeMYgwGGUlyum536p8QJI20uRGZqThcBUK7zavEzlG0hiyM1TpB4GCo2FVSyDH9osLri7GiuqLby59BaHtde_cYwKD-GT46wf04Fwi0K_Ag3s4hnsxktfgdXtnTOVexElOkY5Qj-NiOTGDIw4ZzIZMYVI8x4qNMM-odpxnKiqWFCEBclnjOJFYPHHmGELU8EkJ0hNU2gVdYxIGaFSxtP5cIp_YmBpqmpHJIECZ1lchD0SbBKU2qBzgtoVys7DgWchmjrR4mAIPBBXSHph2o8gOiA68rPBWaMN2JVwnZkdxfE5FXCIjxtt2bG8c1wn6EDQz7OhhKo3BRLs61LozQhksi7QGDcUyJkuDI6xoVF5Kr70VQJvpx830xfdcMNUDNyXfOb6RyFHxjFbjam_6603skUS4yWiBWrN0gbsMXzDMrARELBmceYYW3wUxHE3NAdu0Bp4MTyvCIGO-jYkhJBjKyNGxjG9yHiyB7UO9d3bdoH11B1wn3jFreNpxC0N6pnKOYuJZys2BB1QutHhWE6T-4Gg2bC7D6fYyXFxJErQwlYQbKzEqOR444YWOfhYTwhtJihkb2-dyXFxbCtdO-JbMnUhdNaVDreNNmWM6mzhpKKeJeoPEJrQ1Uq5gkaf1wlVmK4uZQqsUm_HShTIDcgSaWUoGwqnBrmi5SAfAO7nN9iW52LomFO7dOmYSYWRFuZqIVOF6ccv-2dNGiX_M2lwwZYFYTbETzFGx3KNAKFZBKnggZcr2nrF2_h8dv979qItcXFwnxU_-pzhSDRm7GliaF-Sav_-1LbWxRp2Uw7LyWuRR_a7_7XscoXuWzMtFgRVfWj9s2yXgPzsz_kX_Bj1RQTI,">Print view <span class="sr-only">(Opens in new tab)</span></a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
26
tests/data/at/ent_002.json
Normal file
26
tests/data/at/ent_002.json
Normal file
@ -0,0 +1,26 @@
|
||||
{
|
||||
"name": "Bitpanda Payments GmbH",
|
||||
"address": "Jakov-Lind-Straße 2, 1020 Wien, Austria",
|
||||
"phone": "",
|
||||
"email": "",
|
||||
"website": "",
|
||||
"bankIdentificationNumber": "",
|
||||
"commercialRegisterNumber": "",
|
||||
"categories": [
|
||||
"Payment institutions - Payment Institutions licensed in Austria"
|
||||
],
|
||||
"permissions": [
|
||||
{
|
||||
"heading": "§ 1 Abs. 2 Z 3 ZaDiG 2018 - Zahlungsgeschäft",
|
||||
"body": ["Ausführung von Zahlungsvorgängen einschließlich des Transfers von Geldbeträgen auf ein Zahlungskonto beim Zahlungsdienstleister des Zahlungsdienstnutzers oder bei einem anderen Zahlungsdienstleister (Zahlungsgeschäft) a) Ausführung von Lastschriften einschließlich einmaliger Lastschriften (Lastschriftgeschäft); b) Ausführung von Zahlungsvorgängen mittels einer Zahlungskarte oder eines ähnlichen Instruments (Zahlungskartengeschäft); c) Ausführung von Überweisungen einschließlich Daueraufträgen (Überweisungsgeschäft); Die Konzession ist jedoch dahingehend eingeschränkt, dass die angebotenen Zahlungsdienste lediglich von Kunden der Bitpanda GmbH (FN 423018k) in Anspruch genommen werden dürfen;"]
|
||||
},
|
||||
{
|
||||
"heading": "§ 1 Abs. 2 Z 6 ZaDiG 2018 - Finanztransfergeschäft",
|
||||
"body": ["Dienste, bei denen ohne Einrichtung eines Zahlungskontos auf den Namen des Zahlers oder des Zahlungsempfängers ein Geldbetrag eines Zahlers nur zum Transfer eines entsprechenden Betrags an den Zahlungsempfänger oder an einen anderen, im Namen des Zahlungsempfängers handelnden Zahlungsdienstleister entgegengenommen wird oder bei denen der Geldbetrag im Namen des Zahlungsempfängers entgegengenommen und diesem verfügbar gemacht wird (Finanztransfergeschäft) Die Konzession ist jedoch dahingehend eingeschränkt, dass die angebotenen Zahlungsdienste lediglich von Kunden der Bitpanda GmbH (FN 423018k) in Anspruch genommen werden dürfen;"]
|
||||
},
|
||||
{
|
||||
"heading": "§ 1 Abs. 2 Z 7 ZaDiG 2018 - Zahlungsauslösedienst",
|
||||
"body": ["Dienste, die auf Antrag des Zahlungsdienstnutzers einen Zahlungsauftrag in Bezug auf ein bei einem anderen Zahlungsdienstleister geführtes Zahlungskonto auslösen (Zahlungsauslösedienste) Die Konzession ist jedoch dahingehend eingeschränkt, dass die angebotenen Zahlungsdienste lediglich von Kunden der Bitpanda GmbH (FN 423018k) in Anspruch genommen werden dürfen;"]
|
||||
}
|
||||
]
|
||||
}
|
||||
88
tests/data/at/ent_003.html
Normal file
88
tests/data/at/ent_003.html
Normal file
@ -0,0 +1,88 @@
|
||||
<h3>
|
||||
<span class="sr-only">DIMOCO Europe GmbH Brunn am Gebirge</span>
|
||||
<a href="#" class="company-details-toggle" rel="nofollow" title="Show Details" role="presentation"
|
||||
aria-hidden="true">
|
||||
<span class="glyphicon indicator glyphicon-chevron-down" aria-hidden="true"></span>
|
||||
DIMOCO Europe GmbH </a>
|
||||
</h3>
|
||||
|
||||
<div class="company-details">
|
||||
<ul>
|
||||
|
||||
<li><strong>Category:</strong> Payment institutions - Payment Institutions licensed in Austria<br>Payment
|
||||
institutions - Austrian Payment Institutions in EEA (freedom to provide services)<br></li>
|
||||
<li><strong>Address:</strong> Europaring F15/302 | 2345 Brunn am Gebirge | Austria</li>
|
||||
<li><strong>Legal Identifiers:</strong>
|
||||
<ul>
|
||||
<li><strong>Commercial register number:</strong> 199901y</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
<p class="toggle-document-description-wrap"><i class="fa fa-eye" aria-hidden="true"></i> <a href="#"
|
||||
data-toggle="modal"
|
||||
data-target="#document-description-5cf8e5955d032">Show
|
||||
licenses</a></p>
|
||||
<div class="modal fade document-description" id="document-description-5cf8e5955d032" role="dialog" tabindex="-1"
|
||||
aria-labelledby="modal-label-5cf8e5955d032" style="display: none;">
|
||||
<div class="modal-dialog modal-md" role="document">
|
||||
<div class="modal-content">
|
||||
<div class="modal-header"><h6 class="modal-title" id="modal-label-5cf8e5955d032"><span class="sr-only">Description for:</span>
|
||||
Licenses for DIMOCO Europe GmbH - Brunn am Gebirge</h6></div>
|
||||
<div class="modal-body">
|
||||
<h4>§ 1 Abs. 2 Z 3 ZaDiG 2018 - Zahlungsgeschäft</h4>
|
||||
<p>Ausführung von Zahlungsvorgängen einschließlich des Transfers von Geldbeträgen auf ein
|
||||
Zahlungskonto beim Zahlungsdienstleister des Zahlungsdienstnutzers oder bei einem anderen
|
||||
Zahlungsdienstleister (Zahlungsgeschäft) a) Ausführung von Lastschriften einschließlich
|
||||
einmaliger Lastschriften (Lastschriftgeschäft); b) Ausführung von Zahlungsvorgängen mittels
|
||||
einer Zahlungskarte oder eines ähnlichen Instruments (Zahlungskartengeschäft); c) Ausführung von
|
||||
Überweisungen einschließlich Daueraufträgen (Überweisungsgeschäft);</p>
|
||||
<h4>§ 1 Abs. 2 Z 5 ZaDiG
|
||||
2018 - Issuing oder Acquiring</h4>
|
||||
<p>Annahme und Abrechnung von Zahlungsvorgängen (Acquiring);</p>
|
||||
<h4>Zahlungsdienste gemäß Anhang RI
|
||||
2007/64/EG im Rahmen der Dienstleistungsfreiheit in Belgien</h4>
|
||||
<p> 3. Zahlungsgeschäft</p>
|
||||
<h4>Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der
|
||||
Dienstleistungsfreiheit in Belgien</h4>
|
||||
<p> 5. Zahlungsinstrumentegeschäft</p>
|
||||
<h4>Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der
|
||||
Dienstleistungsfreiheit in Belgien (Zusätze/Einschränkungen)</h4>
|
||||
<p>3. Ausführung von Zahlungsvorgängen einschließlich des Transfers von Geldbeträgen auf ein
|
||||
Zahlungskonto beim Zahlungsdienstleister des Nutzers oder bei einem anderen
|
||||
Zahlungsdienstleister:<br>a) Ausführung von Lastschriften einschließlich einmaliger
|
||||
Lastschriften<br>b) Ausführung von Zahlungsvorgängen mittels einer Zahlungskarte oder eines
|
||||
ähnli-chen Instruments<br>c) Ausführung von Überweisungen einschließlich Daueraufträgen<br>5.
|
||||
Annahme und Abrechnung ("acquiring") von Zahlungsinstrumenten (ohne die Gewäh-rung von Krediten
|
||||
entsprechend Artikel 18(4) der RL (EU) 2015/2366)<br></p>
|
||||
<h4>Zahlungsdienste gemäß Anhang RI
|
||||
2007/64/EG im Rahmen der Dienstleistungsfreiheit in Bulgarien</h4>
|
||||
<p> 3. Zahlungsgeschäft</p>
|
||||
<h4>Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der
|
||||
Dienstleistungsfreiheit in Bulgarien</h4>
|
||||
<p> 5. Zahlungsinstrumentegeschäft</p>
|
||||
<h4>Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der
|
||||
Dienstleistungsfreiheit in Bulgarien (Zusätze/Einschränkungen)</h4>
|
||||
<p>3. Ausführung von Zahlungsvorgängen einschließlich des Transfers von Geldbeträgen auf ein
|
||||
Zahlungskonto beim Zahlungsdienstleister des Nutzers oder bei einem anderen
|
||||
Zahlungsdienstleister:<br>a) Ausführung von Lastschriften einschließlich einmaliger
|
||||
Lastschriften<br>b) Ausführung von Zahlungsvorgängen mittels einer Zahlungskarte oder eines
|
||||
ähnli-chen Instruments<br>c) Ausführung von Überweisungen einschließlich Daueraufträgen<br>5.
|
||||
Annahme und Abrechnung ("acquiring") von Zahlungsinstrumenten (ohne die Gewäh-rung von Krediten
|
||||
entsprechend Artikel 18(4) der RL (EU) 2015/2366)<br></p>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-default" data-dismiss="modal">Close description</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="print-view-button-wrap">
|
||||
<i class="fa fa-print" aria-hidden="true" role="presentation"></i>
|
||||
<a class="company-db-print" href="https://www.fma.gv.at/wp-content/plugins/dw-fma/print.php"
|
||||
data-print="eNrtnc1u2zgQgM99C8JcJ_vQOo6TNHWCXDDpxs0G_UXaXnKj5IlERKK8JJVsuuibLNBLgH2JPa1fbIey_BNL2cUu2k5BzCmWSA4_kvONlThB5Gh79Jsd7Yw6kbTQOZCjfX-9N-pYZ1wwXFznwI4G-6POuDTFVBqlE_FisNsfbm37luGo80lN_SsMsD3c2a1fxsrdViMxznNTai1kLk4hUiYBf_8pdilK7cxtfXVcXOJ0StbDtcxhMfHJ2eu3P70V1fwgTvPoZ9-CDbF0kBSmngdHvZO3OWhcJ5S2TrnSqUJb8Vgsbp-t385UDNrCBDuLeu7DyIj-0QNB6j66PRoGGY-PRfcSd2xS5MIVYmqKazUBYcFcXONcXLY3D19vWqRi_2pr1KlvxEbXOzF49uzZ1lwwV_W5WmUNaqt-g-EWbuhhunP01x9iII4j-0RsiwsxFBfyRJ2K7a3BPsJeyDQrdWITsHE6u7t0h30ccjg9wlVcXM7-TPFAEnFd6GXH68IkszudgBaAC4_TTMHsC06diglY8cFIbS_B2GrQKWSTCJyZ3fn-srz0Y5ahrgqNq49A5ctbE4UrcBko68BUAe-36NJ98sGLCTbjSB8PciE1XoN-IEy3sciekD2xucJX0jpsN-rStSwOr3OZqQTj3e_YXbtcXE1wIKLmDC17mCvnILPVOsxqZ6RxMF-kb7Bidpdqj4EjfDaZ0lwnll2trBqg16ePm9PPfo_A3OCmlK3Hd1wiSzB4Rovj6q73X9u7g8P-9KiZWLv3E-vM2tJXgGoRx_EvpfIFYZldWss0B1HqCUYwEKf6Hzapuxy_mvz-UYNIIJ_dzb6IY51KjHR-hiBbT_t7O_3xqcAMO_fzaeFpTlbZ4SOgiSoF5T0WzyFLsHWBKcTwSZtcIt8VYXeFoJZnD9-XBjOttLM7lK8_rtIGc0RfVXnUW5AOnzz6UYrGm_9RJkaHz8_7R7L36OvUhSpa1Hv0tWvA481cIlBNFDdcJ_pvtldRdvEE28XsduRCwU7v3jrWUhKzpEg1CNxWPMQb5H28pHlpYKKq_UPkqY8Lfgbj1BVkYrDf3elVqXj-SnTHH3u-iuz2t4d7e72K7BtcJ3mZJfi8Qux9E4LW_AUPu8_uh-v-CR4E5NJcXFG63wJB6v6Sh91n9wN2H0p_cBnmB6n9bRi0_q-IuAJwBQi3AozxPrH9DQRS82satp6tD9f6F0prau2bDKTeL3BYfBY_YPEx_67wOk5J1W-hoJV_CcT6s_7h6n9qVAVE_d7fzkFaAtaRuAhwEQi5CBSzL5FRTmpN_JHfgyjEpeAeFVcDrgbhVoMzQ_0wsElA6v4chpVn5QNW3pIrb38k5S0rz8oHrrzDE6R91m8g0Eo_p2Hr2fpwrX9pCumItW8ykHq_wGHxWfxwxX8FjvzXepoMpOIvcFh8Fj9g8f2HWK6KqEjf9h9cMKEtAetMXFwHuA6EXFwHHO4GcQXYQCB2v6Jh69n6gK0vf4U8Kk1C6n0Tgtb8BQ-7z-6H6_5rmTlJ6f0GXDCp8xUL-86-h-v7GwXYw_9ECyitb8UgdX-NiCtcMFeAgCtAgRuV0H6T32Sgdb_GYfFZ_HDFf1dktNZvXDCQKl-xsO_se8i-G1cmMqNVfpOB2Po5DovP4ocr_nmJUaj_Rq8FglT9JQ-7z-5Tut_7tnn-Pk5vYELrfpOBVP0FDpvP5of7rv8-K27kFShS8RsMtOLXOCw-ix-2-ED9uN8CQa4-8OM-ux-4-1NJbv4mAq33cxq2nq0P1_oPuDNIpPwXcQ7TMsoU6f_h-Bcg0orQysb1getDuPXho06kIX0o2CQgrVwwcxhWnpUPV_mL2ynQKr9JQKr8HIaVZ-Wple8cfP4b_1Av1g,,">Print
|
||||
view <span class="sr-only">(Opens in new tab)</span></a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
59
tests/data/at/ent_003.json
Normal file
59
tests/data/at/ent_003.json
Normal file
@ -0,0 +1,59 @@
|
||||
{
|
||||
"name": "DIMOCO Europe GmbH",
|
||||
"address": "Europaring F15/302, 2345 Brunn am Gebirge, Austria",
|
||||
"phone": "",
|
||||
"email": "",
|
||||
"website": "",
|
||||
"bankIdentificationNumber": "",
|
||||
"commercialRegisterNumber": "199901y",
|
||||
"categories": [
|
||||
"Payment institutions - Payment Institutions licensed in Austria",
|
||||
"Payment institutions - Austrian Payment Institutions in EEA (freedom to provide services)"
|
||||
],
|
||||
"permissions": [
|
||||
{
|
||||
"heading": "§ 1 Abs. 2 Z 3 ZaDiG 2018 - Zahlungsgeschäft",
|
||||
"body": ["Ausführung von Zahlungsvorgängen einschließlich des Transfers von Geldbeträgen auf ein Zahlungskonto beim Zahlungsdienstleister des Zahlungsdienstnutzers oder bei einem anderen Zahlungsdienstleister (Zahlungsgeschäft) a) Ausführung von Lastschriften einschließlich einmaliger Lastschriften (Lastschriftgeschäft); b) Ausführung von Zahlungsvorgängen mittels einer Zahlungskarte oder eines ähnlichen Instruments (Zahlungskartengeschäft); c) Ausführung von Überweisungen einschließlich Daueraufträgen (Überweisungsgeschäft);"]
|
||||
},
|
||||
{
|
||||
"heading": "§ 1 Abs. 2 Z 5 ZaDiG 2018 - Issuing oder Acquiring",
|
||||
"body": ["Annahme und Abrechnung von Zahlungsvorgängen (Acquiring);"]
|
||||
},
|
||||
{
|
||||
"heading": "Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der Dienstleistungsfreiheit in Belgien",
|
||||
"body": ["3. Zahlungsgeschäft"]
|
||||
},
|
||||
{
|
||||
"heading": "Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der Dienstleistungsfreiheit in Belgien",
|
||||
"body": ["5. Zahlungsinstrumentegeschäft"]
|
||||
},
|
||||
{
|
||||
"heading": "Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der Dienstleistungsfreiheit in Belgien (Zusätze/Einschränkungen)",
|
||||
"body": [
|
||||
"3. Ausführung von Zahlungsvorgängen einschließlich des Transfers von Geldbeträgen auf ein Zahlungskonto beim Zahlungsdienstleister des Nutzers oder bei einem anderen Zahlungsdienstleister:",
|
||||
"a) Ausführung von Lastschriften einschließlich einmaliger Lastschriften",
|
||||
"b) Ausführung von Zahlungsvorgängen mittels einer Zahlungskarte oder eines ähnli-chen Instruments",
|
||||
"c) Ausführung von Überweisungen einschließlich Daueraufträgen",
|
||||
"5. Annahme und Abrechnung (\"acquiring\") von Zahlungsinstrumenten (ohne die Gewäh-rung von Krediten entsprechend Artikel 18(4) der RL (EU) 2015/2366)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"heading": "Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der Dienstleistungsfreiheit in Bulgarien",
|
||||
"body": ["3. Zahlungsgeschäft"]
|
||||
},
|
||||
{
|
||||
"heading": "Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der Dienstleistungsfreiheit in Bulgarien",
|
||||
"body": ["5. Zahlungsinstrumentegeschäft"]
|
||||
},
|
||||
{
|
||||
"heading": "Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der Dienstleistungsfreiheit in Bulgarien (Zusätze/Einschränkungen)",
|
||||
"body": [
|
||||
"3. Ausführung von Zahlungsvorgängen einschließlich des Transfers von Geldbeträgen auf ein Zahlungskonto beim Zahlungsdienstleister des Nutzers oder bei einem anderen Zahlungsdienstleister:",
|
||||
"a) Ausführung von Lastschriften einschließlich einmaliger Lastschriften",
|
||||
"b) Ausführung von Zahlungsvorgängen mittels einer Zahlungskarte oder eines ähnli-chen Instruments",
|
||||
"c) Ausführung von Überweisungen einschließlich Daueraufträgen",
|
||||
"5. Annahme und Abrechnung (\"acquiring\") von Zahlungsinstrumenten (ohne die Gewäh-rung von Krediten entsprechend Artikel 18(4) der RL (EU) 2015/2366)"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
11
tests/data/be/ci_001_fullDetails.json
Normal file
11
tests/data/be/ci_001_fullDetails.json
Normal file
@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "ABN AMRO Private Banking Belgium",
|
||||
"companyType": "Société anonyme",
|
||||
"addressOne": "Kortrijksesteenweg 302",
|
||||
"addressTwo": "9000 Gent",
|
||||
"addressThree": null,
|
||||
"uniqueId": "0415.835.337",
|
||||
"dateOfListing": null,
|
||||
"docLink": null,
|
||||
"normalisedDocLink": null
|
||||
}
|
||||
11
tests/data/be/ci_001_mainDetails.json
Normal file
11
tests/data/be/ci_001_mainDetails.json
Normal file
@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "ABN AMRO Private Banking Belgium",
|
||||
"companyType": "Société anonyme",
|
||||
"addressOne": "Kortrijksesteenweg 302",
|
||||
"addressTwo": "9000 Gent",
|
||||
"addressThree": null,
|
||||
"uniqueId": "0415.835.337",
|
||||
"dateOfListing": null,
|
||||
"docLink": null,
|
||||
"normalisedDocLink": null
|
||||
}
|
||||
1004
tests/data/be/ci_fullpage_001.html
Normal file
1004
tests/data/be/ci_fullpage_001.html
Normal file
File diff suppressed because one or more lines are too long
14
tests/data/be/em_001_fullDetails.json
Normal file
14
tests/data/be/em_001_fullDetails.json
Normal file
@ -0,0 +1,14 @@
|
||||
{
|
||||
"name": "Buy Way Personal Finance",
|
||||
"companyType": "Société anonyme",
|
||||
"addressOne": "Rue de l'Evêque 26",
|
||||
"addressTwo": "1000 Bruxelles",
|
||||
"addressThree": null,
|
||||
"uniqueId": "0400.282.277",
|
||||
"dateOfListing": "07-05-2013",
|
||||
"docLink": "http://www.nbb.be/DOC/CP/ENG/psd/Buy%20Way.docx?t=19418",
|
||||
"normalisedDocLink": "http://www.nbb.be/DOC/CP/ENG/psd/Buy%20Way.docx?t=19418",
|
||||
"activitesGenerales": [ "A", "B" ],
|
||||
"servicesDePaiement": [ "1", "2", "3", "4", "5", "6", "7", "8" ],
|
||||
"autresServicesLies": [ "X" ]
|
||||
}
|
||||
11
tests/data/be/em_001_mainDetails.json
Normal file
11
tests/data/be/em_001_mainDetails.json
Normal file
@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "Ingenico Financial Solutions",
|
||||
"companyType": "Société anonyme",
|
||||
"addressOne": "Leonardo Da Vincilaan 3",
|
||||
"addressTwo": "Corporate Village, Bayreuth Building",
|
||||
"addressThree": "1930 Zaventem",
|
||||
"uniqueId": "0886.476.763",
|
||||
"dateOfListing": "30-11-2012",
|
||||
"docLink": "http://www.nbb.be/DOC/CP/ENG/psd/Ingenico.docx?t=19418",
|
||||
"normalisedDocLink": "http://www.nbb.be/DOC/CP/ENG/psd/Ingenico.docx?t=19418"
|
||||
}
|
||||
509
tests/data/be/em_fullpage_001.html
Normal file
509
tests/data/be/em_fullpage_001.html
Normal file
File diff suppressed because one or more lines are too long
10
tests/data/be/ps_001.html
Normal file
10
tests/data/be/ps_001.html
Normal file
@ -0,0 +1,10 @@
|
||||
<td><strong>Airplus International</strong><br>
|
||||
<em>Société anonyme</em><br>
|
||||
Boulevard de l'Impératrice 66 <br>
|
||||
1000 Bruxelles<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0883.523.807<br>
|
||||
Date de l'inscription à la liste : 26-04-2011<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="http://www.nbb.be/DOC/CP/ENG/psd/BCCC.docx?t=1949" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>2 3 4 5</td>
|
||||
<td>X</td>
|
||||
13
tests/data/be/ps_001.json
Normal file
13
tests/data/be/ps_001.json
Normal file
@ -0,0 +1,13 @@
|
||||
{
|
||||
"name": "Airplus International",
|
||||
"companyType": "Société anonyme",
|
||||
"addressOne": "Boulevard de l'Impératrice 66",
|
||||
"addressTwo": "1000 Bruxelles",
|
||||
"uniqueId": "0883.523.807",
|
||||
"dateOfListing": "26-04-2011",
|
||||
"paymentServices": [
|
||||
"2", "3", "4", "5"
|
||||
],
|
||||
"otherRelatedServices": "X",
|
||||
"docLink": "http://www.nbb.be/DOC/CP/ENG/psd/BCCC.docx?t=1949"
|
||||
}
|
||||
13
tests/data/be/ps_001_fullDetails.json
Normal file
13
tests/data/be/ps_001_fullDetails.json
Normal file
@ -0,0 +1,13 @@
|
||||
{
|
||||
"name": "Airplus International",
|
||||
"companyType": "Société anonyme",
|
||||
"addressOne": "Boulevard de l'Impératrice 66",
|
||||
"addressTwo": "1000 Bruxelles",
|
||||
"addressThree": null,
|
||||
"uniqueId": "0883.523.807",
|
||||
"dateOfListing": "26-04-2011",
|
||||
"docLink": "http://www.nbb.be/DOC/CP/ENG/psd/BCCC.docx?t=19417",
|
||||
"normalisedDocLink": "http://www.nbb.be/DOC/CP/ENG/psd/BCCC.docx?t=19417",
|
||||
"servicesDePaiement": [ "2", "3", "4", "5" ],
|
||||
"autresServicesLies": [ "X" ]
|
||||
}
|
||||
11
tests/data/be/ps_001_mainDetails.json
Normal file
11
tests/data/be/ps_001_mainDetails.json
Normal file
@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "Airplus International",
|
||||
"companyType": "Société anonyme",
|
||||
"addressOne": "Boulevard de l'Impératrice 66",
|
||||
"addressTwo": "1000 Bruxelles",
|
||||
"addressThree": null,
|
||||
"uniqueId": "0883.523.807",
|
||||
"dateOfListing": "26-04-2011",
|
||||
"docLink": "http://www.nbb.be/DOC/CP/ENG/psd/BCCC.docx?t=19417",
|
||||
"normalisedDocLink": "http://www.nbb.be/DOC/CP/ENG/psd/BCCC.docx?t=19417"
|
||||
}
|
||||
751
tests/data/be/ps_fullpage_001.html
Normal file
751
tests/data/be/ps_fullpage_001.html
Normal file
File diff suppressed because one or more lines are too long
500
tests/data/be/ps_index_001.html
Normal file
500
tests/data/be/ps_index_001.html
Normal file
@ -0,0 +1,500 @@
|
||||
<div class="field-name-field-page-intro">
|
||||
<p>Etablissements de paiement agréés en Belgique</p>
|
||||
</div>
|
||||
<div class="field-name-table-of-contents">
|
||||
<h4>
|
||||
<a aria-label="Toggle table of contents" class="icon-table-of-contents" data-toggle="collapse" href="#table-of-contents">Contenu</a>
|
||||
</h4>
|
||||
<nav class="collapse in" id="table-of-contents">
|
||||
<ul>
|
||||
<li>
|
||||
<a href="#bm_Header_0">Etablissements de paiement de droit belge</a>
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
|
||||
</div>
|
||||
<div class="description">
|
||||
Article 8 §1, 1° de la loi du 11 mars 2018 relative au statut et au contrôle des établissements de paiement et des établissements de monnaie électronique, à l'accès à l'activité de prestataire de services de paiement, et à l'activité d'émission de monnaie électronique, et à l'accès aux systèmes de paiement
|
||||
</div>
|
||||
<div class="legend">
|
||||
<h3>(*) Services de paiement</h3>
|
||||
<ol>
|
||||
<li>Les services permettant de verser des espèces sur un compte de paiement et toutes les opérations qu’exige la gestion d’un compte de paiement;</li>
|
||||
<li>Les services permettant de retirer des espèces d’un compte de paiement et toutes les opérations qu’exige la gestion d’un compte de paiement;</li>
|
||||
<li>L’exécution d’opérations de paiement, y compris les transferts de fonds sur un compte de paiement auprès du prestataire de services de paiement de l’utilisateur ou auprès d’un autre prestataire de services de paiement :
|
||||
<ol style="list-style-type:lower-alpha">
|
||||
<li>l’exécution de prélèvements, y compris de prélèvements autorisés unitairement,</li>
|
||||
<li>l’exécution d’opérations de paiement à l’aide d’une carte de paiement ou d’un dispositif similaire,</li>
|
||||
<li>l’exécution de virements, y compris d’ordres permanents;</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>L’exécution d’opérations de paiement dans le cadre desquelles les fonds sont couverts par une ligne de crédit accordée à l’utilisateur de services de paiement :
|
||||
<ol style="list-style-type:lower-alpha">
|
||||
<li>l’exécution de prélèvements, y compris de prélèvements autorisés unitairement,</li>
|
||||
<li>l’exécution d’opérations de paiement à l’aide d’une carte de paiement ou d’un dispositif similaire,</li>
|
||||
<li>l’exécution de virements, y compris d’ordres permanents;</li>
|
||||
</ol>
|
||||
</li>
|
||||
<li>L’émission d’instruments de paiement et/ou l’acquisition d’opérations de paiement;</li>
|
||||
<li>Les transmissions de fonds;</li>
|
||||
<li>Les services d’initiation de paiement;</li>
|
||||
<li>Les services d’information sur les comptes;</li>
|
||||
</ol>
|
||||
<h3>(**) Autres services liés</h3>
|
||||
<ol style="list-style-type:upper-alpha">
|
||||
<li value="24">Octroi de crédits liés aux services de paiement visés aux points 4, 5 ou 7 des services de paiement visés sous (*)</li>
|
||||
</ol>
|
||||
|
||||
</div>
|
||||
<h2>
|
||||
Situation au 08-04-2019
|
||||
</h2>
|
||||
<a href="#bm_Changes12M">Modifications de la liste au cours des douze derniers mois</a>
|
||||
<ul class="List1">
|
||||
<li>
|
||||
<h3 id="bm_Header_0">
|
||||
Etablissements de paiement de droit belge
|
||||
</h3>
|
||||
<div class="number-of-records">
|
||||
Nombre total d'établissements : 24
|
||||
</div>
|
||||
<div class="table-responsive">
|
||||
<table class="table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th></th>
|
||||
<th>Services de paiement<div class="stars">(*)</div></th>
|
||||
<th>Autres services liés<div class="stars">(**)</div></th>
|
||||
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td><strong>Airplus International</strong><br>
|
||||
<em>Société anonyme</em><br>
|
||||
Boulevard de l'Impératrice 66 <br>
|
||||
1000 Bruxelles<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0883.523.807<br>
|
||||
Date de l'inscription à la liste : 26-04-2011<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="http://www.nbb.be/DOC/CP/ENG/psd/BCCC.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>2 3 4 5</td>
|
||||
<td>X</td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>Alpha Card</strong><br>
|
||||
<em>Société coopérative à responsabilité limitée</em><br>
|
||||
Boulevard du Souverain 100 <br>
|
||||
1170 Watermael Boitsfort<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0463.926.551<br>
|
||||
Date de l'inscription à la liste : 26-04-2011<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="http://www.nbb.be/DOC/CP/ENG/psd/Alpha%20Card.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>2 3 4 5</td>
|
||||
<td>X</td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>Alpha Card Merchant Services</strong><br>
|
||||
<em>Société coopérative à responsabilité limitée</em><br>
|
||||
Boulevard du Souverain 100 <br>
|
||||
1170 Watermael Boitsfort<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0475.933.171<br>
|
||||
Date de l'inscription à la liste : 26-04-2011<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="http://www.nbb.be/DOC/CP/ENG/psd/Alpha%20Card%20Merchant%20Services.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>2 3 4 5</td>
|
||||
<td>X</td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>Belmoney Transfert</strong><br>
|
||||
<em>Société privée à responsabilité limitée</em><br>
|
||||
Place Bara 28 <br>
|
||||
1070 Bruxelles<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0540.745.997<br>
|
||||
Date de l'inscription à la liste : 20-03-2018<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="http://www.nbb.be/DOC/CP/ENG/psd/Belmoney.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>6</td>
|
||||
<td></td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>Cashfree</strong><br>
|
||||
<em>Société anonyme</em><br>
|
||||
Mortelputstraat 49 <br>
|
||||
9830 Sint-Martens-Latem<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0663.774.859<br>
|
||||
Date de l'inscription à la liste : 11-07-2017</td>
|
||||
<td>3 4 5</td>
|
||||
<td></td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>Cofidis</strong><br>
|
||||
<em>Société anonyme</em><br>
|
||||
Chaussée de Lille 422A <br>
|
||||
7501 Orcq<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0400.359.283<br>
|
||||
Date de l'inscription à la liste : 07-05-2013</td>
|
||||
<td>4 5 6</td>
|
||||
<td>X</td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>Digiteal</strong><br>
|
||||
<em>Société anonyme</em><br>
|
||||
rue Emile Francqui 6 <br>
|
||||
1435 Corbais<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0630.675.588<br>
|
||||
Date de l'inscription à la liste : 31-10-2017<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="http://www.nbb.be/DOC/CP/ENG/psd/Teal%20IT.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>3 7 8</td>
|
||||
<td></td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>Ebury Partners Belgium</strong><br>
|
||||
<em>Société anonyme</em><br>
|
||||
Boulevard du Régent 37 <br>
|
||||
1000 Bruxelles<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0681.746.187<br>
|
||||
Date de l'inscription à la liste : 31-10-2017<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="file:///L:/PRXNWEBP/DOC/CP/eng/psd/Ebury.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>3 4 6</td>
|
||||
<td></td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>eDebex</strong><br>
|
||||
<em>Société anonyme</em><br>
|
||||
Rue Jules Cockx 8-10 <br>
|
||||
1160 Bruxelles<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0502.697.352<br>
|
||||
Date de l'inscription à la liste : 02-02-2016<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="http://www.nbb.be/DOC/CP/ENG/psd/Edebex.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>3</td>
|
||||
<td></td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>EPBF</strong><br>
|
||||
<em>Société anonyme</em><br>
|
||||
Chaussée de la Hulpe 181 <br>
|
||||
1170 Watermael-Boitsfort<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0837.808.497<br>
|
||||
Date de l'inscription à la liste : 27-06-2011<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="http://www.nbb.be/DOC/CP/ENG/psd/EPBF.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>3 4 5</td>
|
||||
<td>X</td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>Gold Commodities Forex</strong><br>
|
||||
<em>Société anonyme</em><br>
|
||||
Winderickxplein 3 bte 2<br>
|
||||
1652 Beersel<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0832.602.171<br>
|
||||
Date de l'inscription à la liste : 27-06-2011<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="http://www.nbb.be/DOC/CP/ENG/psd/GCF.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>3 6</td>
|
||||
<td></td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>HomeSend</strong><br>
|
||||
<em>Société coopérative à responsabilité limitée</em><br>
|
||||
Rue des Colonies 56 <br>
|
||||
1000 Bruxelles<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0549.987.921<br>
|
||||
Date de l'inscription à la liste : 09-02-2016<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="http://www.nbb.be/DOC/CP/ENG/psd/HomeSend.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>3 4</td>
|
||||
<td></td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>iBanFirst</strong><br>
|
||||
<em>Société anonyme</em><br>
|
||||
Avenue Louise 350 <br>
|
||||
1050 Bruxelles<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0849.872.824<br>
|
||||
Date de l'inscription à la liste : 22-05-2013<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="http://www.nbb.be/DOC/CP/ENG/psd/FX4Biz.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>3 4 7 8</td>
|
||||
<td></td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>Isabel</strong><br>
|
||||
<em>Société anonyme</em><br>
|
||||
Boulevard de l'Impératrice 13-15 <br>
|
||||
1000 Bruxelles<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0455.530.509<br>
|
||||
Date de l'inscription à la liste : 19-02-2019<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="file:///L:/PRXNWEBP/DOC/CP/eng/psd/Isabel.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>7 8</td>
|
||||
<td></td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>Let's Didid</strong><br>
|
||||
<em>Société anonyme</em><br>
|
||||
Montagne du Parc 3 <br>
|
||||
1000 Bruxelles<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0450.355.261<br>
|
||||
Date de l'inscription à la liste : 19-02-2019</td>
|
||||
<td>7 8</td>
|
||||
<td></td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>Money International</strong><br>
|
||||
<em>Société anonyme</em><br>
|
||||
Chaussée de Gand 100 <br>
|
||||
1080 Bruxelles<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0475.445.104<br>
|
||||
Date de l'inscription à la liste : 26-03-2019</td>
|
||||
<td>6</td>
|
||||
<td></td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>MoneyGram International</strong><br>
|
||||
<em>Société privée à responsabilité limitée</em><br>
|
||||
rue Joseph Stevens 7 <br>
|
||||
1000 Bruxelles<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0671.690.653<br>
|
||||
Date de l'inscription à la liste : 19-12-2017<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="http://www.nbb.be/DOC/CP/ENG/psd/MoneyGram.xlsx?t=19411" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>6</td>
|
||||
<td></td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>Moneytrans Payment Services</strong><br>
|
||||
<em>Société anonyme</em><br>
|
||||
Boulevard de Waterloo 77 <br>
|
||||
1000 Bruxelles<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0449.356.557<br>
|
||||
Date de l'inscription à la liste : 26-04-2011<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="http://www.nbb.be/DOC/CP/ENG/psd/change_center_sprl.xlsx?t=19411" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>3 6</td>
|
||||
<td></td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>Oonex</strong><br>
|
||||
<em>Société anonyme</em><br>
|
||||
Avenue Louise 367 <br>
|
||||
1050 Bruxelles<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0849.185.510<br>
|
||||
Date de l'inscription à la liste : 23-08-2016<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="http://www.nbb.be/DOC/CP/ENG/psd/Oonex.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>3 5</td>
|
||||
<td></td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>PAY-NXT </strong><br>
|
||||
<em>Société anonyme</em><br>
|
||||
avenue Reine Astrid 92A <br>
|
||||
1310 La Hulpe<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0649.860.804<br>
|
||||
Date de l'inscription à la liste : 11-10-2016<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="http://www.nbb.be/DOC/CP/ENG/psd/Pay%20Nxt.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>3 5</td>
|
||||
<td></td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>Transferwise Europe</strong><br>
|
||||
<em>Société anonyme</em><br>
|
||||
Square de Meeûs 38 bte 40<br>
|
||||
1000 Bruxelles<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0713.629.988<br>
|
||||
Date de l'inscription à la liste : 19-03-2019<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="file:///L:/PRXNWEBP/DOC/CP/eng/psd/TransferWise%20Europe.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>3 5 6</td>
|
||||
<td></td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>Travelex</strong><br>
|
||||
<em>Société anonyme</em><br>
|
||||
Steendam 108 <br>
|
||||
9000 Gent<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0451.299.329<br>
|
||||
Date de l'inscription à la liste : 26-04-2011<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="http://www.nbb.be/DOC/CP/ENG/psd/Travelex.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>6</td>
|
||||
<td></td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>Worldline</strong><br>
|
||||
<em>Société anonyme</em><br>
|
||||
Chaussée de Haecht 1442 <br>
|
||||
Blue Star Building - FIN - D4
|
||||
<br>
|
||||
1130 Bruxelles<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0418.547.872<br>
|
||||
Date de l'inscription à la liste : 21-09-2010<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="http://www.nbb.be/DOC/CP/ENG/psd/Atos_Worldline_SA.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>3 4 5 7 8</td>
|
||||
<td></td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td><strong>WorldRemit Belgium</strong><br>
|
||||
<em>Société anonyme</em><br>
|
||||
Place Marcel Broodthaers 8 <br>
|
||||
1060 Bruxelles<br>
|
||||
<br>
|
||||
Numéro d'identification unique : 0718.634.495<br>
|
||||
Date de l'inscription à la liste : 26-03-2019<br>
|
||||
Succursales/Agents/Libre prestation des services : <a href="file:///L:/PRXNWEBP/DOC/CP/eng/psd/WorldRemit.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
|
||||
<td>6</td>
|
||||
<td></td>
|
||||
|
||||
</tr>
|
||||
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
</div>
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
<h2 id="bm_Changes12M">
|
||||
Modifications de la liste au cours des douze derniers mois
|
||||
</h2>
|
||||
<div class="changes-12">
|
||||
<ul>
|
||||
<li>
|
||||
<h3>
|
||||
Etablissements de paiement de droit belge
|
||||
</h3>
|
||||
<ul>
|
||||
<li>
|
||||
<span class="ChangeTypeTitle">Agrément</span>
|
||||
<div class="table-responsive">
|
||||
<table class="table" width="100%" cellpadding="5">
|
||||
<tbody>
|
||||
<tr>
|
||||
<th class="ColumnTitleModification" width="10%" style="word-wrap:normal;word-break:keep-all">Date effective</th>
|
||||
<th class="ColumnTitleModification" width="90%" style="word-wrap:normal;word-break:keep-all"> </th>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="white-space:nowrap">19-02-2019</td>
|
||||
<td style="word-wrap:normal;word-break:keep-all">Isabel </td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="white-space:nowrap">19-02-2019</td>
|
||||
<td style="word-wrap:normal;word-break:keep-all">Let's Didid </td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="white-space:nowrap">19-03-2019</td>
|
||||
<td style="word-wrap:normal;word-break:keep-all">Transferwise Europe </td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="white-space:nowrap">26-03-2019</td>
|
||||
<td style="word-wrap:normal;word-break:keep-all">Money International </td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="white-space:nowrap">26-03-2019</td>
|
||||
<td style="word-wrap:normal;word-break:keep-all">WorldRemit Belgium </td>
|
||||
|
||||
</tr>
|
||||
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</li>
|
||||
<li>
|
||||
<span class="ChangeTypeTitle">Radiation</span>
|
||||
<div class="table-responsive">
|
||||
<table class="table" width="100%" cellpadding="5">
|
||||
<tbody>
|
||||
<tr>
|
||||
<th class="ColumnTitleModification" width="10%" style="word-wrap:normal;word-break:keep-all">Date effective</th>
|
||||
<th class="ColumnTitleModification" width="90%" style="word-wrap:normal;word-break:keep-all"> </th>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="white-space:nowrap">26-06-2018</td>
|
||||
<td style="word-wrap:normal;word-break:keep-all">B+S Payment Europe </td>
|
||||
|
||||
</tr>
|
||||
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</li>
|
||||
<li>
|
||||
<span class="ChangeTypeTitle">Changement de la dénomination sociale</span>
|
||||
<div class="table-responsive">
|
||||
<table class="table" width="100%" cellpadding="5">
|
||||
<tbody>
|
||||
<tr>
|
||||
<th class="ColumnTitleModification" width="10%" style="word-wrap:normal;word-break:keep-all">Date effective</th>
|
||||
<th class="ColumnTitleModification" width="30%" style="word-wrap:normal;word-break:keep-all">Ancienne dénomination</th>
|
||||
<th class="ColumnTitleModification" width="60%" style="word-wrap:normal;word-break:keep-all">Nouvelle dénomination</th>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="white-space:nowrap">04-01-2019</td>
|
||||
<td style="word-wrap:normal;word-break:keep-all">Teal IT </td>
|
||||
<td style="word-wrap:normal;word-break:keep-all">Digiteal </td>
|
||||
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="white-space:nowrap">01-02-2019</td>
|
||||
<td style="word-wrap:normal;word-break:keep-all">Bank Card Company Corporate </td>
|
||||
<td style="word-wrap:normal;word-break:keep-all">Airplus International </td>
|
||||
|
||||
</tr>
|
||||
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
|
||||
</div>
|
||||
<p><em>Fin de liste</em></p>
|
||||
<a href="#PrudentialList">Haut de liste</a>
|
||||
1324
tests/data/nl/din329_d2_01.html
Normal file
1324
tests/data/nl/din329_d2_01.html
Normal file
File diff suppressed because one or more lines are too long
42
tests/data/nl/din329_d2_01.json
Normal file
42
tests/data/nl/din329_d2_01.json
Normal file
@ -0,0 +1,42 @@
|
||||
[
|
||||
[
|
||||
"Statutory name",
|
||||
"NIBC Bank N.V."
|
||||
],
|
||||
[
|
||||
"Trade name",
|
||||
"NIBC Direct"
|
||||
],
|
||||
[
|
||||
"Address",
|
||||
"Carnegieplein 4"
|
||||
],
|
||||
[
|
||||
"Postal code",
|
||||
"2517 KJ"
|
||||
],
|
||||
[
|
||||
"Place of residence",
|
||||
"'S-GRAVENHAGE"
|
||||
],
|
||||
[
|
||||
"Country",
|
||||
"Netherlands"
|
||||
],
|
||||
[
|
||||
"Disclosure",
|
||||
""
|
||||
],
|
||||
[
|
||||
"Chamber of Commerce",
|
||||
"27032036"
|
||||
],
|
||||
[
|
||||
"Relation number DNB",
|
||||
"B0685"
|
||||
],
|
||||
[
|
||||
"Category",
|
||||
"Emittent effecten CSDB, Bank"
|
||||
]
|
||||
]
|
||||
21
tests/data/no/cb_001_ps.html
Normal file
21
tests/data/no/cb_001_ps.html
Normal file
@ -0,0 +1,21 @@
|
||||
<h3 class="license-unit-label">Cross-border services/classes</h3>
|
||||
<p>
|
||||
The entity may provide cross-border services/classes from Slovakia
|
||||
</p>
|
||||
|
||||
|
||||
<div>
|
||||
<h4 class="license-gov-subheading">To Norway without physical establishment, for the following services/classes</h4>
|
||||
<ul>
|
||||
<li>
|
||||
Payment service activity
|
||||
<ul>
|
||||
<li>
|
||||
Service 3 and 4 in annex to PSD 1
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</div>
|
||||
16
tests/data/no/cb_001_ps.json
Normal file
16
tests/data/no/cb_001_ps.json
Normal file
@ -0,0 +1,16 @@
|
||||
{
|
||||
"name": "Cross-border services/classes The entity may provide cross-border services/classes from Slovakia",
|
||||
"data": [
|
||||
{
|
||||
"name": "To Norway without physical establishment, for the following services/classes",
|
||||
"data": [
|
||||
{
|
||||
"name": "Payment service activity",
|
||||
"data": [
|
||||
"Service 3 and 4 in annex to PSD 1"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
36
tests/data/no/cb_002_ps.html
Normal file
36
tests/data/no/cb_002_ps.html
Normal file
@ -0,0 +1,36 @@
|
||||
<h3 class="license-unit-label">Cross-border services/classes</h3>
|
||||
<p>
|
||||
The entity may provide cross-border services/classes from United Kingdom
|
||||
</p>
|
||||
|
||||
|
||||
<div>
|
||||
<h4 class="license-gov-subheading">To Norway without physical establishment, for the following services/classes</h4>
|
||||
<ul>
|
||||
<li>
|
||||
Payment service activity
|
||||
<ul>
|
||||
<li>
|
||||
Service 6 in annex to PSD 1
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div>
|
||||
<h4 class="license-gov-subheading">To Norway through tied agents</h4>
|
||||
<ul>
|
||||
<li class="license-link-item">
|
||||
<a href="?id=192475">HASHYL FINANCIAL SERVICES YAHYA SONKO</a> <span>(Norway)</span>
|
||||
</li>
|
||||
<li class="license-link-item">
|
||||
<a href="?id=192481">SHARANHA GULLSMED Varathalingam Selliah</a> <span>(Norway)</span>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
23
tests/data/no/cb_002_ps.json
Normal file
23
tests/data/no/cb_002_ps.json
Normal file
@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "Cross-border services/classes The entity may provide cross-border services/classes from United Kingdom",
|
||||
"data": [
|
||||
{
|
||||
"name": "To Norway without physical establishment, for the following services/classes",
|
||||
"data": [
|
||||
{
|
||||
"name": "Payment service activity",
|
||||
"data": [
|
||||
"Service 6 in annex to PSD 1"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "To Norway through tied agents",
|
||||
"data": [
|
||||
"HASHYL FINANCIAL SERVICES YAHYA SONKO (Norway)",
|
||||
"SHARANHA GULLSMED Varathalingam Selliah (Norway)"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
24
tests/data/no/cb_003_ps.html
Normal file
24
tests/data/no/cb_003_ps.html
Normal file
@ -0,0 +1,24 @@
|
||||
<h3 class="license-unit-label">Cross-border services/classes</h3>
|
||||
<p>
|
||||
The entity may provide cross-border services/classes from United Kingdom
|
||||
</p>
|
||||
|
||||
|
||||
<div>
|
||||
<h4 class="license-gov-subheading">To Norway without physical establishment, for the following services/classes</h4>
|
||||
<ul>
|
||||
<li>
|
||||
Payment service activity
|
||||
<ul>
|
||||
<li>
|
||||
Service 3 and 4 in annex to PSD 1
|
||||
</li>
|
||||
<li>
|
||||
Service 5 in annex to PSD 1
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</div>
|
||||
17
tests/data/no/cb_003_ps.json
Normal file
17
tests/data/no/cb_003_ps.json
Normal file
@ -0,0 +1,17 @@
|
||||
{
|
||||
"name": "Cross-border services/classes The entity may provide cross-border services/classes from United Kingdom",
|
||||
"data": [
|
||||
{
|
||||
"name": "To Norway without physical establishment, for the following services/classes",
|
||||
"data": [
|
||||
{
|
||||
"name": "Payment service activity",
|
||||
"data": [
|
||||
"Service 3 and 4 in annex to PSD 1",
|
||||
"Service 5 in annex to PSD 1"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
33
tests/data/no/cb_004_ps.html
Normal file
33
tests/data/no/cb_004_ps.html
Normal file
@ -0,0 +1,33 @@
|
||||
<h3 class="license-unit-label">Cross-border services/classes</h3>
|
||||
<p>
|
||||
The entity may provide cross-border services/classes from United Kingdom
|
||||
</p>
|
||||
|
||||
|
||||
<div>
|
||||
<h4 class="license-gov-subheading">To Norway without physical establishment, for the following services/classes</h4>
|
||||
<ul>
|
||||
<li>
|
||||
Payment service activity
|
||||
<ul>
|
||||
<li>
|
||||
Service 1 and 2 in annex to PSD 1
|
||||
</li>
|
||||
<li>
|
||||
Service 3 and 4 in annex to PSD 1
|
||||
</li>
|
||||
<li>
|
||||
Service 5 in annex to PSD 1
|
||||
</li>
|
||||
<li>
|
||||
Service 6 in annex to PSD 1
|
||||
</li>
|
||||
<li>
|
||||
Service 7 in annex to PSD 1
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</div>
|
||||
20
tests/data/no/cb_004_ps.json
Normal file
20
tests/data/no/cb_004_ps.json
Normal file
@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "Cross-border services/classes The entity may provide cross-border services/classes from United Kingdom",
|
||||
"data": [
|
||||
{
|
||||
"name": "To Norway without physical establishment, for the following services/classes",
|
||||
"data": [
|
||||
{
|
||||
"name": "Payment service activity",
|
||||
"data": [
|
||||
"Service 1 and 2 in annex to PSD 1",
|
||||
"Service 3 and 4 in annex to PSD 1",
|
||||
"Service 5 in annex to PSD 1",
|
||||
"Service 6 in annex to PSD 1",
|
||||
"Service 7 in annex to PSD 1"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
32
tests/data/no/cb_005_ps_with_empty_item.html
Normal file
32
tests/data/no/cb_005_ps_with_empty_item.html
Normal file
@ -0,0 +1,32 @@
|
||||
<h3 class="license-unit-label">Cross-border services/classes</h3>
|
||||
<p>
|
||||
The entity may provide cross-border services/classes from Germany
|
||||
</p>
|
||||
|
||||
|
||||
<div>
|
||||
<h4 class="license-gov-subheading">To Norway without physical establishment, for the following services/classes</h4>
|
||||
<ul>
|
||||
<li>
|
||||
|
||||
<ul>
|
||||
<li>
|
||||
Distribution/Redemption of electronic money
|
||||
</li>
|
||||
<li>
|
||||
Issuing electronic money
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li>
|
||||
Payment service activity
|
||||
<ul>
|
||||
<li>
|
||||
Service 5 in annex to PSD 1
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</div>
|
||||
23
tests/data/no/cb_005_ps_with_empty_item.json
Normal file
23
tests/data/no/cb_005_ps_with_empty_item.json
Normal file
@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "Cross-border services/classes The entity may provide cross-border services/classes from Germany",
|
||||
"data": [
|
||||
{
|
||||
"name": "To Norway without physical establishment, for the following services/classes",
|
||||
"data": [
|
||||
{
|
||||
"name": "",
|
||||
"data": [
|
||||
"Distribution/Redemption of electronic money",
|
||||
"Issuing electronic money"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Payment service activity",
|
||||
"data": [
|
||||
"Service 5 in annex to PSD 1"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
71
tests/data/no/cb_006_em_with_multi_countries.html
Normal file
71
tests/data/no/cb_006_em_with_multi_countries.html
Normal file
@ -0,0 +1,71 @@
|
||||
<h3 class="license-unit-label">Cross-border services/classes</h3>
|
||||
<p>
|
||||
The entity may provide cross-border services/classes from Norway
|
||||
</p>
|
||||
|
||||
|
||||
<div>
|
||||
<h4 class="license-gov-subheading">To multiple countries without physical establishment, for the following services/classes</h4>
|
||||
<ul>
|
||||
<li>
|
||||
|
||||
<ul>
|
||||
<li>
|
||||
Distribution/Redemption of electronic money
|
||||
<ul>
|
||||
<li>
|
||||
Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li>
|
||||
Issuing electronic money
|
||||
<ul>
|
||||
<li>
|
||||
Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li>
|
||||
Payment service activity
|
||||
<ul>
|
||||
<li>
|
||||
Service 1 and 2 in annex to PSD 1
|
||||
<ul>
|
||||
<li>
|
||||
Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li>
|
||||
Service 3 and 4 in annex to PSD 1
|
||||
<ul>
|
||||
<li>
|
||||
Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li>
|
||||
Service 5 in annex to PSD 1
|
||||
<ul>
|
||||
<li>
|
||||
Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
<li>
|
||||
Service 6 in annex to PSD 1
|
||||
<ul>
|
||||
<li>
|
||||
Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
||||
</div>
|
||||
44
tests/data/no/cb_006_em_with_multi_countries.json
Normal file
44
tests/data/no/cb_006_em_with_multi_countries.json
Normal file
@ -0,0 +1,44 @@
|
||||
{
|
||||
"name": "Cross-border services/classes The entity may provide cross-border services/classes from Norway",
|
||||
"data": [
|
||||
{
|
||||
"name": "To multiple countries without physical establishment, for the following services/classes",
|
||||
"data": [
|
||||
{
|
||||
"name": "",
|
||||
"data": [
|
||||
{
|
||||
"name": "Distribution/Redemption of electronic money",
|
||||
"data": [ "Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom" ]
|
||||
},
|
||||
{
|
||||
"name": "Issuing electronic money",
|
||||
"data": [ "Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom" ]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Payment service activity",
|
||||
"data": [
|
||||
{
|
||||
"name": "Service 1 and 2 in annex to PSD 1",
|
||||
"data": [ "Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom" ]
|
||||
},
|
||||
{
|
||||
"name": "Service 3 and 4 in annex to PSD 1",
|
||||
"data": [ "Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom" ]
|
||||
},
|
||||
{
|
||||
"name": "Service 5 in annex to PSD 1",
|
||||
"data": [ "Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom" ]
|
||||
},
|
||||
{
|
||||
"name": "Service 6 in annex to PSD 1",
|
||||
"data": [ "Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom" ]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -1,15 +1,12 @@
|
||||
{
|
||||
"crossborderServicesclasses": [
|
||||
"The entity may provide cross-border services/classes from Spain"
|
||||
],
|
||||
"name": "Cross-border services/classes The entity may provide cross-border services/classes from Spain",
|
||||
"data": [
|
||||
{
|
||||
"description": {
|
||||
"main": "main",
|
||||
"ancillaryServicesMiFidDirective": "Ancillary services (MiFid directive)",
|
||||
"investmentServicesAndActivitiesMiFidDirective": "Investment services and activities (MiFid directive)"
|
||||
},
|
||||
"main": [
|
||||
"name": "To Norway without physical establishment, for the following services/classes",
|
||||
"data": [
|
||||
{
|
||||
"name": "",
|
||||
"data": [
|
||||
"1. Acceptance of deposits and other repayable funds",
|
||||
"2. Lending",
|
||||
"3. Financial leasing",
|
||||
@ -22,8 +19,11 @@
|
||||
"8. Participation in securities issues and the provision of services related to such issues",
|
||||
"9. Advisory services",
|
||||
"11. Portfolio management and advice"
|
||||
],
|
||||
"ancillaryServicesMiFidDirective": [
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Ancillary services (MiFid directive)",
|
||||
"data": [
|
||||
"No 1: Safekeeping and administration of financial instruments",
|
||||
"No 2: Granting credits or loans",
|
||||
"No 3: Advice on capital structure, strategy et al.; advice and services re. to mergers and purchases",
|
||||
@ -31,8 +31,11 @@
|
||||
"No 5: Provision of investment recommendations, analyses and general recommendations",
|
||||
"No 6: Services related to underwriting",
|
||||
"No 7: Services related to underlying of derivatives connected to investment and ancillary services"
|
||||
],
|
||||
"investmentServicesAndActivitiesMiFidDirective": [
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Investment services and activities (MiFid directive)",
|
||||
"data": [
|
||||
"No 1: Reception and transmission of orders",
|
||||
"No 2: Execution of orders",
|
||||
"No 3: Dealing on own account",
|
||||
@ -44,3 +47,5 @@
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@ -12,17 +12,14 @@
|
||||
"1/1/2016"
|
||||
],
|
||||
"crossBorder": {
|
||||
"crossborderServicesclasses": [
|
||||
"The entity may provide cross-border services/classes from Spain"
|
||||
],
|
||||
"name": "Cross-border services/classes The entity may provide cross-border services/classes from Spain",
|
||||
"data": [
|
||||
{
|
||||
"description": {
|
||||
"main": "main",
|
||||
"ancillaryServicesMiFidDirective": "Ancillary services (MiFid directive)",
|
||||
"investmentServicesAndActivitiesMiFidDirective": "Investment services and activities (MiFid directive)"
|
||||
},
|
||||
"main": [
|
||||
"name": "To Norway without physical establishment, for the following services/classes",
|
||||
"data": [
|
||||
{
|
||||
"name": "",
|
||||
"data": [
|
||||
"1. Acceptance of deposits and other repayable funds",
|
||||
"2. Lending",
|
||||
"3. Financial leasing",
|
||||
@ -35,8 +32,11 @@
|
||||
"8. Participation in securities issues and the provision of services related to such issues",
|
||||
"9. Advisory services",
|
||||
"11. Portfolio management and advice"
|
||||
],
|
||||
"ancillaryServicesMiFidDirective": [
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Ancillary services (MiFid directive)",
|
||||
"data": [
|
||||
"No 1: Safekeeping and administration of financial instruments",
|
||||
"No 2: Granting credits or loans",
|
||||
"No 3: Advice on capital structure, strategy et al.; advice and services re. to mergers and purchases",
|
||||
@ -44,8 +44,11 @@
|
||||
"No 5: Provision of investment recommendations, analyses and general recommendations",
|
||||
"No 6: Services related to underwriting",
|
||||
"No 7: Services related to underlying of derivatives connected to investment and ancillary services"
|
||||
],
|
||||
"investmentServicesAndActivitiesMiFidDirective": [
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Investment services and activities (MiFid directive)",
|
||||
"data": [
|
||||
"No 1: Reception and transmission of orders",
|
||||
"No 2: Execution of orders",
|
||||
"No 3: Dealing on own account",
|
||||
@ -57,5 +60,7 @@
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@ -11,18 +11,16 @@
|
||||
"registrationDate": [
|
||||
"1/1/2016"
|
||||
],
|
||||
|
||||
"crossBorder": {
|
||||
"crossborderServicesclasses": [
|
||||
"The entity may provide cross-border services/classes from Luxembourg"
|
||||
],
|
||||
"name": "Cross-border services/classes The entity may provide cross-border services/classes from Luxembourg",
|
||||
"data": [
|
||||
{
|
||||
"description": {
|
||||
"main": "main",
|
||||
"ancillaryServicesMiFidDirective": "Ancillary services (MiFid directive)",
|
||||
"investmentServicesAndActivitiesMiFidDirective": "Investment services and activities (MiFid directive)"
|
||||
},
|
||||
"main": [
|
||||
"name": "To Norway without physical establishment, for the following services/classes",
|
||||
"data": [
|
||||
{
|
||||
"name": "",
|
||||
"data": [
|
||||
"1. Acceptance of deposits and other repayable funds",
|
||||
"2. Lending",
|
||||
"3. Financial leasing",
|
||||
@ -41,15 +39,21 @@
|
||||
"12. Safekeeping and administration of securities",
|
||||
"13. Credit reference services",
|
||||
"14. Safe custody services"
|
||||
],
|
||||
"ancillaryServicesMiFidDirective": [
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Ancillary services (MiFid directive)",
|
||||
"data": [
|
||||
"No 1: Safekeeping and administration of financial instruments",
|
||||
"No 2: Granting credits or loans",
|
||||
"No 4: Foreign exchange services connected to the provision of investment services",
|
||||
"No 5: Provision of investment recommendations, analyses and general recommendations",
|
||||
"No 7: Services related to underlying of derivatives connected to investment and ancillary services"
|
||||
],
|
||||
"investmentServicesAndActivitiesMiFidDirective": [
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "Investment services and activities (MiFid directive)",
|
||||
"data": [
|
||||
"No 1: Reception and transmission of orders",
|
||||
"No 2: Execution of orders",
|
||||
"No 4: Active management of investors' portfolios",
|
||||
@ -58,6 +62,8 @@
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"licenseDescription": "Authorisation to be registered in a Norwegian securities register, register of shareholders and/or unit holder register on behalf of the beneficial owner. A foreign nominee is in addition subject to supervision by the relevant supervisory authority in its home country.",
|
||||
|
||||
@ -1,13 +1,9 @@
|
||||
{
|
||||
"crossborderServicesclasses": [
|
||||
"The entity may provide cross-border services/classes from Sweden"
|
||||
],
|
||||
"name": "Cross-border services/classes The entity may provide cross-border services/classes from Sweden",
|
||||
"data": [
|
||||
{
|
||||
"description": {
|
||||
"toNorwayThroughABranchEstablishment": "To Norway through a branch establishment"
|
||||
},
|
||||
"toNorwayThroughABranchEstablishment": [
|
||||
"name": "To Norway through a branch establishment",
|
||||
"data": [
|
||||
"AVIDA FINANS AB NUF (Norway)"
|
||||
]
|
||||
}
|
||||
|
||||
@ -12,19 +12,16 @@
|
||||
"8/4/2008"
|
||||
],
|
||||
"crossBorder": {
|
||||
"crossborderServicesclasses": [
|
||||
"The entity may provide cross-border services/classes from Sweden"
|
||||
],
|
||||
"name": "Cross-border services/classes The entity may provide cross-border services/classes from Sweden",
|
||||
"data": [
|
||||
{
|
||||
"description": {
|
||||
"toNorwayThroughABranchEstablishment": "To Norway through a branch establishment"
|
||||
},
|
||||
"toNorwayThroughABranchEstablishment": [
|
||||
"name": "To Norway through a branch establishment",
|
||||
"data": [
|
||||
"AVIDA FINANS AB NUF (Norway)"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
}
|
||||
]
|
||||
|
||||
@ -1,21 +1,20 @@
|
||||
{
|
||||
"crossborderServicesclasses": [
|
||||
"The entity may provide cross-border services/classes from United Kingdom"
|
||||
],
|
||||
"name": "Cross-border services/classes The entity may provide cross-border services/classes from United Kingdom",
|
||||
"data": [
|
||||
{
|
||||
"description": {
|
||||
"paymentServiceActivity": "Payment service activity"
|
||||
},
|
||||
"paymentServiceActivity": [
|
||||
"name": "To Norway without physical establishment, for the following services/classes",
|
||||
"data": [
|
||||
{
|
||||
"name": "Payment service activity",
|
||||
"data": [
|
||||
"Service 6 in annex to PSD 1"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": {
|
||||
"toNorwayThroughTiedAgents": "To Norway through tied agents"
|
||||
},
|
||||
"toNorwayThroughTiedAgents": [
|
||||
"name": "To Norway through tied agents",
|
||||
"data": [
|
||||
"HASHYL FINANCIAL SERVICES YAHYA SONKO (Norway)",
|
||||
"SHARANHA GULLSMED Varathalingam Selliah (Norway)"
|
||||
]
|
||||
|
||||
@ -12,23 +12,22 @@
|
||||
"1/1/2016"
|
||||
],
|
||||
"crossBorder": {
|
||||
"crossborderServicesclasses": [
|
||||
"The entity may provide cross-border services/classes from United Kingdom"
|
||||
],
|
||||
"name": "Cross-border services/classes The entity may provide cross-border services/classes from United Kingdom",
|
||||
"data": [
|
||||
{
|
||||
"description": {
|
||||
"paymentServiceActivity": "Payment service activity"
|
||||
},
|
||||
"paymentServiceActivity": [
|
||||
"name": "To Norway without physical establishment, for the following services/classes",
|
||||
"data": [
|
||||
{
|
||||
"name": "Payment service activity",
|
||||
"data": [
|
||||
"Service 6 in annex to PSD 1"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"description": {
|
||||
"toNorwayThroughTiedAgents": "To Norway through tied agents"
|
||||
},
|
||||
"toNorwayThroughTiedAgents": [
|
||||
"name": "To Norway through tied agents",
|
||||
"data": [
|
||||
"HASHYL FINANCIAL SERVICES YAHYA SONKO (Norway)",
|
||||
"SHARANHA GULLSMED Varathalingam Selliah (Norway)"
|
||||
]
|
||||
|
||||
52
tests/scrape.at.js
Normal file
52
tests/scrape.at.js
Normal file
@ -0,0 +1,52 @@
|
||||
const tape = require('tape');
|
||||
const _test = require('tape-promise').default; // <---- notice 'default'
|
||||
const test = _test(tape); // decorate tape
|
||||
|
||||
const diff = require('deep-diff');
|
||||
const fs = require('fs');
|
||||
const jsonfile = require('jsonfile');
|
||||
|
||||
const Austria = require('../ncas/at');
|
||||
|
||||
const atScraper = new Austria();
|
||||
|
||||
test.test('Austria 🇦🇹 :: Tests', async t => {
|
||||
|
||||
t.test('Test extractEntityDetails ent_001', async t => {
|
||||
const html = fs.readFileSync('tests/data/at/ent_001.html');
|
||||
const output = await atScraper.extractEntityDetails(html);
|
||||
const expectedJSON = jsonfile.readFileSync('tests/data/at/ent_001.json');
|
||||
|
||||
t.deepEquals(output, expectedJSON, 'Extracted entity details from Page');
|
||||
|
||||
console.log(diff(output, expectedJSON));
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test('Test extractEntityDetails ent_002', async t => {
|
||||
const html = fs.readFileSync('tests/data/at/ent_002.html');
|
||||
const output = await atScraper.extractEntityDetails(html);
|
||||
const expectedJSON = jsonfile.readFileSync('tests/data/at/ent_002.json');
|
||||
|
||||
t.deepEquals(output, expectedJSON, 'Extracted entity details from Page');
|
||||
|
||||
console.log(diff(output, expectedJSON));
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test('Test extractEntityDetails ent_003', async t => {
|
||||
const html = fs.readFileSync('tests/data/at/ent_003.html');
|
||||
const output = await atScraper.extractEntityDetails(html);
|
||||
const expectedJSON = jsonfile.readFileSync('tests/data/at/ent_003.json');
|
||||
|
||||
t.deepEquals(output, expectedJSON, 'Extracted entity details from Page');
|
||||
|
||||
console.log(diff(output, expectedJSON));
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.end();
|
||||
});
|
||||
166
tests/scrape.be.js
Normal file
166
tests/scrape.be.js
Normal file
@ -0,0 +1,166 @@
|
||||
const cheerio = require('cheerio');
|
||||
const tape = require('tape');
|
||||
const _test = require('tape-promise').default; // <---- notice 'default'
|
||||
const test = _test(tape); // decorate tape
|
||||
|
||||
const fs = require('fs');
|
||||
const jsonfile = require('jsonfile');
|
||||
|
||||
const Belgium = require('../ncas/be');
|
||||
const beScraper = new Belgium();
|
||||
|
||||
test.test('Entities', async t => {
|
||||
t.test('Extract main details...', async t => {
|
||||
t.test('...from td container', async t => {
|
||||
const htmlFile = 'tests/data/be/ps_fullpage_001.html';
|
||||
const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' });
|
||||
const $ = cheerio.load(html, { 'decodeEntities': false });
|
||||
const detailsContainer = $('ul.List1 div.table-responsive tbody tr td').eq(0);
|
||||
|
||||
const output = await beScraper.extractMainDetails(detailsContainer);
|
||||
|
||||
const expectedJSON = jsonfile.readFileSync('tests/data/be/ps_001_mainDetails.json');
|
||||
t.deepEquals(output, expectedJSON);
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test('...from li container', async t => {
|
||||
const htmlFile = 'tests/data/be/ci_fullpage_001.html';
|
||||
const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' });
|
||||
const $ = cheerio.load(html, { 'decodeEntities': false });
|
||||
const detailsContainer = $('ul.List1 ul.List2 > li > ul > li').eq(0);
|
||||
|
||||
const output = await beScraper.extractMainDetails(detailsContainer);
|
||||
|
||||
const expectedJSON = jsonfile.readFileSync('tests/data/be/ci_001_mainDetails.json');
|
||||
t.deepEquals(output, expectedJSON);
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test('...from unusual entity (3-line address and large spacing)', async t => {
|
||||
const htmlFile = 'tests/data/be/em_fullpage_001.html';
|
||||
const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' });
|
||||
const $ = cheerio.load(html, { 'decodeEntities': false });
|
||||
const detailsContainer = $('ul.List1 div.table-responsive tbody tr').eq(4).children('td').eq(0);
|
||||
|
||||
const output = await beScraper.extractMainDetails(detailsContainer);
|
||||
|
||||
const expectedJSON = jsonfile.readFileSync('tests/data/be/em_001_mainDetails.json');
|
||||
t.deepEquals(output, expectedJSON);
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test('Extract full details...', async t => {
|
||||
t.test('...from payment service', async t => {
|
||||
const htmlFile = 'tests/data/be/ps_fullpage_001.html';
|
||||
const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' });
|
||||
const $ = cheerio.load(html, { 'decodeEntities': false });
|
||||
const fullDetailsContainer = $('ul.List1 div.table-responsive tbody tr').eq(0);
|
||||
|
||||
const output = await beScraper.extractFullDetails(fullDetailsContainer, 0);
|
||||
|
||||
const expectedJSON = jsonfile.readFileSync('tests/data/be/ps_001_fullDetails.json');
|
||||
t.deepEquals(output, expectedJSON);
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test('...from emoney service', async t => {
|
||||
const htmlFile = 'tests/data/be/em_fullpage_001.html';
|
||||
const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' });
|
||||
const $ = cheerio.load(html, { 'decodeEntities': false });
|
||||
const fullDetailsContainer = $('ul.List1 div.table-responsive tbody tr').eq(0);
|
||||
|
||||
const output = await beScraper.extractFullDetails(fullDetailsContainer, 0);
|
||||
|
||||
const expectedJSON = jsonfile.readFileSync('tests/data/be/em_001_fullDetails.json');
|
||||
t.deepEquals(output, expectedJSON);
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test('...from credit institution', async t => {
|
||||
const htmlFile = 'tests/data/be/ci_fullpage_001.html';
|
||||
const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' });
|
||||
const $ = cheerio.load(html, { 'decodeEntities': false });
|
||||
const fullDetailsContainer = $('ul.List1 ul.List2 > li > ul > li').eq(0);
|
||||
|
||||
const output = await beScraper.extractFullDetails(fullDetailsContainer, 2);
|
||||
|
||||
const expectedJSON = jsonfile.readFileSync('tests/data/be/ci_001_fullDetails.json');
|
||||
t.deepEquals(output, expectedJSON);
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test('Extract entities from container...', async t => {
|
||||
t.test('...of payment services (tbody)', async t => {
|
||||
const htmlFile = 'tests/data/be/ps_fullpage_001.html';
|
||||
const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' });
|
||||
const $ = cheerio.load(html, { 'decodeEntities': false });
|
||||
const entitiesContainer = $('ul.List1 tbody');
|
||||
|
||||
const output = await beScraper.extractEntitiesFromContainer(entitiesContainer, 0);
|
||||
|
||||
t.equals(output.length, 24);
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test('...of credit institutions (ul)', async t => {
|
||||
const htmlFile = 'tests/data/be/ci_fullpage_001.html';
|
||||
const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' });
|
||||
const $ = cheerio.load(html, { 'decodeEntities': false });
|
||||
const entitiesContainer = $('ul.List1 ul.List2 > li > ul').eq(0); // get the first list only for this test
|
||||
|
||||
const output = await beScraper.extractEntitiesFromContainer(entitiesContainer, 2);
|
||||
|
||||
t.equals(output.length, 25);
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test('Extract index...', async t => {
|
||||
t.test('...of payment services', async t => {
|
||||
const htmlFile = 'tests/data/be/ps_fullpage_001.html';
|
||||
const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' });
|
||||
const $ = cheerio.load(html, { 'decodeEntities': false });
|
||||
const indexContainer = $('#PrudentialList');
|
||||
|
||||
const output = await beScraper.extractIndex(indexContainer, 0);
|
||||
|
||||
// console.log(output);
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test('...of credit institutions', async t => {
|
||||
const htmlFile = 'tests/data/be/ci_fullpage_001.html';
|
||||
const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' });
|
||||
const $ = cheerio.load(html, { 'decodeEntities': false });
|
||||
const indexContainer = $('#PrudentialList');
|
||||
|
||||
const output = await beScraper.extractIndex(indexContainer, 2);
|
||||
|
||||
// console.log(output);
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.end();
|
||||
});
|
||||
@ -222,59 +222,52 @@ test('FRANCE Breaking CI 001', async t => {
|
||||
});
|
||||
|
||||
test('FRANCE:: DIR-3741', async t => {
|
||||
const dir3741_001 = fs.readFileSync('tests/data/fr/dir3741_001.html');
|
||||
const dir3741_001Data = jsonfile.readFileSync('tests/data/fr/dir3741_001.json');
|
||||
const dir3741001 = fs.readFileSync('tests/data/fr/dir3741_001.html');
|
||||
const dir3741001Data = jsonfile.readFileSync('tests/data/fr/dir3741_001.json');
|
||||
|
||||
const dir3741_002 = fs.readFileSync('tests/data/fr/dir3741_002.html');
|
||||
const dir3741_002Data = jsonfile.readFileSync('tests/data/fr/dir3741_002.json');
|
||||
const dir3741002 = fs.readFileSync('tests/data/fr/dir3741_002.html');
|
||||
const dir3741002Data = jsonfile.readFileSync('tests/data/fr/dir3741_002.json');
|
||||
|
||||
const dir3741_003 = fs.readFileSync('tests/data/fr/dir3741_003.html');
|
||||
const dir3741_003Data = jsonfile.readFileSync('tests/data/fr/dir3741_003.json');
|
||||
const dir3741003 = fs.readFileSync('tests/data/fr/dir3741_003.html');
|
||||
const dir3741003Data = jsonfile.readFileSync('tests/data/fr/dir3741_003.json');
|
||||
|
||||
const frScraper = new France();
|
||||
t.test('FRANCE::Extract Details from Page 1/103', async t => {
|
||||
const $ = cheerio.load(dir3741_001);
|
||||
const $ = cheerio.load(dir3741001);
|
||||
const $table = $('table.table tr');
|
||||
|
||||
const links = await frScraper.extractLinks($table, true);
|
||||
|
||||
|
||||
|
||||
const linkCount = links.length;
|
||||
|
||||
t.equal(linkCount, 1, 'Scrapes the correct number of links (1)');
|
||||
t.deepEquals(links, dir3741_001Data, 'Links match the data');
|
||||
t.deepEquals(links, dir3741001Data, 'Links match the data');
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test('FRANCE::Extract Details from Page 4/103', async t => {
|
||||
const $ = cheerio.load(dir3741_002);
|
||||
const $ = cheerio.load(dir3741002);
|
||||
const $table = $('table.table tr');
|
||||
|
||||
const links = await frScraper.extractLinks($table, true);
|
||||
|
||||
|
||||
|
||||
const linkCount = links.length;
|
||||
|
||||
t.equal(linkCount, 3, 'Scrapes the correct number of links (3)');
|
||||
t.deepEquals(links, dir3741_002Data, 'Links match the data');
|
||||
t.deepEquals(links, dir3741002Data, 'Links match the data');
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test('FRANCE::Extract Details from Page 11/103', async t => {
|
||||
const $ = cheerio.load(dir3741_003);
|
||||
const $ = cheerio.load(dir3741003);
|
||||
const $table = $('table.table tr');
|
||||
|
||||
const links = await frScraper.extractLinks($table, true);
|
||||
|
||||
|
||||
|
||||
|
||||
const linkCount = links.length;
|
||||
|
||||
t.equal(linkCount, 2, 'Scrapes the correct number of links (2)');
|
||||
t.deepEquals(links, dir3741_003Data, 'Links match the data');
|
||||
t.deepEquals(links, dir3741003Data, 'Links match the data');
|
||||
t.end();
|
||||
});
|
||||
|
||||
|
||||
@ -315,3 +315,19 @@ test('NL:: Scrape a Credit Service', async t => {
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
test('NL:: DIN-329 defects', async t => {
|
||||
const defect2AHTML = fs.readFileSync('tests/data/nl/din329_d2_01.html');
|
||||
|
||||
t.test('NL::Defect 2', async t => {
|
||||
const expectedJSON = jsonfile.readFileSync('tests/data/nl/din329_d2_01.json');
|
||||
|
||||
const output = await nlScraper.extractDetail(defect2AHTML);
|
||||
|
||||
t.deepEquals(output, expectedJSON, 'Extracted Details from Page correctly');
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
@ -225,6 +225,77 @@ test.test('Entity', async t => {
|
||||
t.end();
|
||||
});
|
||||
|
||||
test.test('NO:: Cross-Border format', async t => {
|
||||
|
||||
t.test('NO:: Extract Cross-Border services 001 (PS)', async t => {
|
||||
const crossborderHtml = fs.readFileSync('tests/data/no/cb_001_ps.html').toString();
|
||||
const expectedJSON = jsonfile.readFileSync('tests/data/no/cb_001_ps.json');
|
||||
|
||||
const output = await noScraper.recurseCrossborderHtml(crossborderHtml);
|
||||
|
||||
t.deepEquals(output, expectedJSON, 'Extracted cross-border services from html');
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test('NO:: Extract Cross-Border services 002 (PS)', async t => {
|
||||
const crossborderHtml = fs.readFileSync('tests/data/no/cb_002_ps.html').toString();
|
||||
const expectedJSON = jsonfile.readFileSync('tests/data/no/cb_002_ps.json');
|
||||
|
||||
const output = await noScraper.recurseCrossborderHtml(crossborderHtml);
|
||||
|
||||
t.deepEquals(output, expectedJSON, 'Extracted cross-border services from html');
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test('NO:: Extract Cross-Border services 003 (PS)', async t => {
|
||||
const crossborderHtml = fs.readFileSync('tests/data/no/cb_003_ps.html').toString();
|
||||
const expectedJSON = jsonfile.readFileSync('tests/data/no/cb_003_ps.json');
|
||||
|
||||
const output = await noScraper.recurseCrossborderHtml(crossborderHtml);
|
||||
|
||||
t.deepEquals(output, expectedJSON, 'Extracted cross-border services from html');
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test('NO:: Extract Cross-Border services 004 (PS)', async t => {
|
||||
const crossborderHtml = fs.readFileSync('tests/data/no/cb_004_ps.html').toString();
|
||||
const expectedJSON = jsonfile.readFileSync('tests/data/no/cb_004_ps.json');
|
||||
|
||||
const output = await noScraper.recurseCrossborderHtml(crossborderHtml);
|
||||
|
||||
t.deepEquals(output, expectedJSON, 'Extracted cross-border services from html');
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test('NO:: Extract Cross-Border services 005 (PS with an empty list item)', async t => {
|
||||
const crossborderHtml = fs.readFileSync('tests/data/no/cb_005_ps_with_empty_item.html').toString();
|
||||
const expectedJSON = jsonfile.readFileSync('tests/data/no/cb_005_ps_with_empty_item.json');
|
||||
|
||||
const output = await noScraper.recurseCrossborderHtml(crossborderHtml);
|
||||
|
||||
t.deepEquals(output, expectedJSON, 'Extracted cross-border services from html');
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.test('NO:: Extract Cross-Border services 006 (PS with multiple countries)', async t => {
|
||||
const crossborderHtml = fs.readFileSync('tests/data/no/cb_006_em_with_multi_countries.html').toString();
|
||||
const expectedJSON = jsonfile.readFileSync('tests/data/no/cb_006_em_with_multi_countries.json');
|
||||
|
||||
const output = await noScraper.recurseCrossborderHtml(crossborderHtml);
|
||||
|
||||
t.deepEquals(output, expectedJSON, 'Extracted cross-border services from html');
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
t.end();
|
||||
});
|
||||
|
||||
|
||||
@ -1 +1 @@
|
||||
VERSION=1.0.1
|
||||
VERSION=1.0.2
|
||||
|
||||
Loading…
Reference in New Issue
Block a user