martin/obdfcascrape
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

final update

Browse Source
This commit is contained in:
Martin Donnelly 2019-08-15 08:48:49 +01:00
parent 105a2c0abe
commit 534fd67b5d
67 changed files with 6526 additions and 949 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -159,3 +159,4 @@ artefacts/*
/archive.tar.gz
/user/
/zip

7
Jenkinsfile vendored
View File

@ -1,6 +1,3 @@
@Library('OpenBankingUK/ob-pipeline-library') _
@Library('OpenBankingUK/devdir-jenkins-pipeline-lib') _
javaMsPipelinev2 {
projectName='obdfcascrape'
cluster='nca'
}
buildNodeMsPipeline {projectName='obdfcascrape'}

10
ecosystem.config.js
View File

@ -25,14 +25,14 @@ function buildApps() {
{ 'cron':'DE_CRON', 'start':'DE', 'name':'DE', 'script':'de.js', 'proxy': 'de', 'crontime': '0 0 * * *' }, // 03:55:38 - de free at 4:00
{ 'cron':'NL_CRON', 'start':'NL', 'name':'NL', 'script':'nl.js', 'proxy': 'nl', 'crontime': '0 0 * * *' }, // 07:23:19 - nl free at 7:30
{ 'cron':'PL_CRON', 'start':'PL', 'name':'PL', 'script':'pl.js', 'proxy': 'ch', 'crontime': '0 0 * * *' }, // 17:59:18 - ch free at 18:00
{ 'cron':'DK_CRON', 'start':'DK', 'name':'DK', 'script':'dk.js', 'proxy': 'uk' },
{ 'cron':'ES_CRON', 'start':'ES', 'name':'ES', 'script':'es.js', 'proxy': 'uk' },
{ 'cron':'LV_CRON', 'start':'LV', 'name':'LV', 'script':'lv.js', 'proxy': 'nl', 'crontime': '30 7 * * *' }, // 13:56.232 - nl free at 7:45
{ 'cron':'DK_CRON', 'start':'DK', 'name':'DK', 'script':'dk.js', 'proxy': 'de', 'crontime': '0 4 * * *' }, // 11:08.616 - de free at 4:15
{ 'cron':'ES_CRON', 'start':'ES', 'name':'ES', 'script':'es.js', 'proxy': 'de', 'crontime': '15 4 * * *' }, // 36:44.523- de free at 4:55
{ 'cron':'EE_CRON', 'start':'EE', 'name':'EE', 'script':'ee.js', 'proxy': 'de', 'crontime': '0 5 * * *' }, // 05:22:04.226 - de free after 10:30
{ 'cron':'NO_CRON', 'start':'NO', 'name':'NO', 'script':'no.js', 'proxy': 'fr', 'crontime': '0 4 * * *' }, // 05:12:57.792 - fr free after 9:20
{ 'cron':'GI_CRON', 'start':'GI', 'name':'GI', 'script':'gi.js', 'proxy': 'uk' },
{ 'cron':'GR_CRON', 'start':'GR', 'name':'GR', 'script':'gr.js', 'proxy': 'uk' },
{ 'cron':'MT_CRON', 'start':'MT', 'name':'MT', 'script':'mt.js', 'proxy': 'uk' },
{ 'cron':'LV_CRON', 'start':'LV', 'name':'LV', 'script':'lv.js', 'proxy': 'uk' },
{ 'cron':'NO_CRON', 'start':'NO', 'name':'NO', 'script':'no.js', 'proxy': 'uk' },
{ 'cron':'EE_CRON', 'start':'EE', 'name':'EE', 'script':'ee.js', 'proxy': 'uk' },
{ 'cron':'BG_CRON', 'start':'BG', 'name':'BG', 'script':'bg.js', 'proxy': 'uk' },
{ 'cron':'AT_CRON', 'start':'AT', 'name':'AT', 'script':'at.js', 'proxy': 'uk' },
{ 'cron':'FI_CRON', 'start':'FI', 'name':'FI', 'script':'fi.js', 'proxy': 'uk' },

177
helpers/scraper.js
View File

@ -52,7 +52,7 @@ const useDig = ['cy'];
// Default the region
AWS.config.update({ 'region': 'eu-west-1' });
if (process.env.NODE_ENV !== 'production')
if (process.env.NODE_ENV !== 'production')
AWS.config.update({ 'accessKeyId': process.env.AWS_ACCESS_KEY_ID, 'secretAccessKey': process.env.AWS_SECRET_ACCESS_KEY, 'region': process.env.AWS_REGION || 'eu-west-1' });
const s3 = new AWS.S3();
@ -143,7 +143,7 @@ class Scraper extends EventEmitter {
*/
async emptyPath(path) {
if (process.env.NODE_ENV === 'production')
if (process.env.NODE_ENV === 'production')
await del([path]).then(paths => {
logger.warn('Deleted files and folders:\n', paths.join('\n'));
});
@ -165,7 +165,7 @@ class Scraper extends EventEmitter {
await this._createDirectory(this.path);
await this._createDirectory(this.debugPath);
}
/**
* 'Human' like click delay
* @returns {number}
@ -183,7 +183,7 @@ class Scraper extends EventEmitter {
async _killRunningBrowser() {
// if (typeof(this.browser) !== 'undefined' && this.browser !== null) {
if (this.browser)
if (this.browser)
try{
logger.info('Trying to close hanging / running browser');
@ -231,14 +231,17 @@ class Scraper extends EventEmitter {
'--disable-gpu',
'--window-size=1920x1080',
'--hide-scrollbars',
'--disable-default-apps',
'--remote-debugging-port=9222'
'--disable-default-apps'
]
}).catch((err) => {
logger.error('Puppeteer failed to launch');
logger.error(err);
});
const browserVersion = await this.browser.version();
logger.info(`Browser version ${browserVersion}`);
this.browser.on('disconnected', () => {
logger.warn('Browser has become detached!');
@ -253,7 +256,7 @@ class Scraper extends EventEmitter {
async _forcePageClose() {
// if (this.page !== null) {
if (this.page)
if (this.page)
try{
logger.warn('Browser Page exists: DESTROYING');
@ -309,9 +312,11 @@ class Scraper extends EventEmitter {
this.page.on('error', async err => {
logger.warn('Page crashed', err);
await this._uploadError();
logger.warn('page.onError::emit recover');
this.emit('recover');
if (!this.detatchable) {
await this._uploadError();
logger.warn('page.onError::emit recover');
this.emit('recover');
}
});
this.page.on('pageerror', async err => {
@ -399,9 +404,9 @@ class Scraper extends EventEmitter {
* @private
*/
async _makeScreenshot(page, destPath, waitFor = null) {
if (waitFor)
if (waitFor)
await page.waitFor(waitFor);
await page.setViewport({ 'width': 1200, 'height': 800 });
await page.screenshot({ 'path': `artefacts/screenshots/${destPath}.png`, 'fullPage': true }).catch((err) => {
logger.error('Screenshot', err);
@ -449,7 +454,7 @@ class Scraper extends EventEmitter {
*/
async _randomWait(page, minTime = 2, maxTime = 10, msg = '') {
const insertedMsg = (msg.length > 0) ? `${this.id} ${msg} - ` : `${this.id} `;
const waitTime = Math.floor(Math.random() * (maxTime - minTime + 1) + minTime);
logger.debug(`${insertedMsg}Waiting ${waitTime} seconds...`);
await page.waitFor(waitTime * 1000);
@ -497,9 +502,9 @@ class Scraper extends EventEmitter {
return new Promise((resolve, reject) => {
const fullPath = `${__dirname}/../artefacts/${destPath}`;
fs.writeFile(fullPath, data, function(err) {
if(err)
if(err)
reject(err);
else
else
resolve(`File saved to '${fullPath}'`);
});
});
@ -579,11 +584,11 @@ class Scraper extends EventEmitter {
'zlib': { 'level': 9 } // Sets the compression level.
});
if (glob)
if (glob)
archive.glob(`${destPath}`);
else
else
archive.directory(`${destPath}/`);
archive.finalize().then(() => {
logger.debug('Archive finished');
resolve();
@ -595,6 +600,7 @@ class Scraper extends EventEmitter {
*
* @param destPath
* @param filename
* @param glob
* @returns {Promise<*>}
* @private
*/
@ -693,7 +699,7 @@ class Scraper extends EventEmitter {
async _getWhoIsJSON(destPath = null, withPrefix = false) {
const options = { };
if (!destPath)
if (!destPath)
throw new Error('No destination path');
const explodedURL = this.explodeURL(destPath);
@ -973,7 +979,7 @@ class Scraper extends EventEmitter {
.then(async exists => {
console.log(`file exists: ${exists}`);
if (exists)
if (exists)
await fs.renameSync(origFN, newFN);
}).catch((e) => {
logger.error(e);
@ -1008,7 +1014,7 @@ class Scraper extends EventEmitter {
await this._checkS3FileExists(langFileName)
.then(exists => {
if (exists)
if (exists)
return new Promise((resolve, reject) => { // (*)
this._getFileS3(langFileName).then((data) => {
this.dictionary = new Map(JSON.parse(data));
@ -1030,7 +1036,7 @@ class Scraper extends EventEmitter {
async _saveDictionary() {
if (this.dictionary.size > 0) {
logger.debug('Save dictionary', this.dictionary.size);
return new Promise((resolve, reject) => {
const langFileName = `lang.${this.id.toLowerCase()}.json`;
const arrayedMap = JSON.stringify([...this.dictionary]);
@ -1099,6 +1105,9 @@ class Scraper extends EventEmitter {
async _done() {
logger.info('<=- DONE -=>');
// OK To close the browser window now
this.canDetach();
const now = new Date();
this.perf.finished = now.getTime();
@ -1114,9 +1123,6 @@ class Scraper extends EventEmitter {
await this._archive();
// OK To close the browser window now
this.canDetach();
await this._forcePageClose();
await this._killRunningBrowser();
@ -1307,9 +1313,9 @@ class Scraper extends EventEmitter {
};
return new Promise((resolve, reject) => {
if (filename === null)
if (filename === null)
return reject(Error('No filename for S3'));
s3.headObject(params).promise().then((i) => {
logger.debug(`${filename} exists`);
@ -1366,7 +1372,7 @@ class Scraper extends EventEmitter {
*/
async _sendMessage(id, msg, msgBody = 'New upload') {
logger.debug('+ _sendMessage', process.env.SQS_ID);
if (typeof process.env.SQS_ID !== 'undefined' && process.env.SQS_ID !== null)
if (typeof process.env.SQS_ID !== 'undefined' && process.env.SQS_ID !== null)
try {
const sqs = new AWS.SQS({ 'apiVersion': '2012-11-05' });
@ -1394,7 +1400,7 @@ class Scraper extends EventEmitter {
return data;
}).catch((err) => {
logger.error(err);
return err;
});
}
@ -1469,7 +1475,7 @@ class Scraper extends EventEmitter {
* @private
*/
async _doNonRepudiation(skip = false, options = {}) {
if (!skip)
if (!skip)
try{
if (typeof this.startPage === 'undefined' || this.startPage === null)
@ -1506,12 +1512,24 @@ class Scraper extends EventEmitter {
const key = localStorage.key(i);
json[key] = localStorage.getItem(key);
}
return json;
});
await jsonfile.writeFileSync(filePath, json);
}
async _getLocalStorage( ) {
return await this.page.evaluate(() => {
const json = {};
for (let i = 0; i < localStorage.length; i++) {
const key = localStorage.key(i);
json[key] = localStorage.getItem(key);
}
return json;
});
}
_checkFileExistsSync(filePath) {
try {
fs.accessSync(filePath, fs.F_OK);
@ -1568,7 +1586,7 @@ class Scraper extends EventEmitter {
if (err.message.indexOf('net::ERR_FAILED') !== -1)
this.browserCrashed = true;
if (!noRecover)
if (!noRecover)
this.emit('recover');
});
}
@ -1606,7 +1624,7 @@ class Scraper extends EventEmitter {
*/
_throttle (callback, limit) {
var wait = false;
return function () {
if (!wait) {
callback.apply(null, arguments);
@ -1628,13 +1646,13 @@ class Scraper extends EventEmitter {
_once(func) {
var alreadyCalled = false;
var result;
return function() {
if (!alreadyCalled) {
result = func.apply(this, arguments);
alreadyCalled = true;
}
return result;
};
};
@ -1664,39 +1682,54 @@ class Scraper extends EventEmitter {
async __recover(restartURL) {
logger.warn(`*** RECONNECTING ${this.id} PAGE ***`);
let crashCount = 0;
if (this.crashLog.has(this.lastUrl)) {
let crashCount = this.crashLog.get(this.lastUrl);
crashCount = this.crashLog.get(this.lastUrl);
crashCount++;
this.crashLog.set(this.lastUrl, crashCount);
if (crashCount >= 3)
if (crashCount >= 3)
logger.error('The page has crashed more than 3 times', this.lastUrl);
if (crashCount >= 10) {
logger.error('10 times on the same page is enough', this.lastUrl);
return;
}
}
else
else
this.crashLog.set(this.lastUrl, 1);
if (this.browserCrashed) await this._initBrowser(true);
if (crashCount < 10) {
if (this.browserCrashed) await this._initBrowser(true);
await this._createBrowserPage();
await this._createBrowserPage();
logger.debug('Reattach processNewPage', (typeof this.processNewPage === 'function') ? 'Yes' : 'No');
if (typeof this.processNewPage === 'function')
this.page.on('domcontentloaded', () => {
this.processNewPage();
});
logger.debug('Reattach processNewPage', (typeof this.processNewPage === 'function') ? 'Yes' : 'No');
if (typeof this.processNewPage === 'function')
this.page.on('domcontentloaded', () => {
this.processNewPage();
});
const antiCollision = 125 + (Math.floor(Math.random() * (15 - 1)) * 500);
const timeout = 90000 + antiCollision;
const onHold = (crashCount >= 3) ? (90000 * crashCount) : 0;
const antiCollision = 125 + (Math.floor(Math.random() * (15 - 1)) * 500);
const timeout = 90000 + antiCollision + onHold;
logger.info(`🚨 Restarting in ${(timeout / 1000).toFixed(2)} seconds.`);
logger.info(`🚨 Restarting in ${(timeout / 1000).toFixed(2)} seconds.`);
setTimeout(async() => {
logger.warn('Attempting recovery..');
setTimeout(async() => {
logger.warn(`Attempting recovery to ${restartURL}`);
await this.restart(restartURL);
}, timeout);
await this.restart(restartURL);
}, timeout);
}
}
/**
*
* @param restartURL
* @returns {Promise<void>}
*/
async restart(restartURL) {
const rURL = restartURL || this.lastUrl;
logger.info(`Restarting ${this.id} // Going to ${rURL}`);
@ -1704,6 +1737,12 @@ class Scraper extends EventEmitter {
await this._goto(rURL);
}
/**
*
* @param filename
* @param data
* @returns {Promise<void>}
*/
async saveFile(filename, data) {
try{
fs.writeFileSync(filename, data);
@ -1713,6 +1752,40 @@ class Scraper extends EventEmitter {
}
}
/**
*
* @param s
* @returns {string}
*/
soundex(s) {
const a = s.toLowerCase().split(''),
codes = {
'a': '', 'e': '', 'i': '', 'o': '', 'u': '',
'b': 1, 'f': 1, 'p': 1, 'v': 1,
'c': 2, 'g': 2, 'j': 2, 'k': 2, 'q': 2, 's': 2, 'x': 2, 'z': 2,
'd': 3, 't': 3,
'l': 4,
'm': 5, 'n': 5,
'r': 6
};
const f = a.shift();
let r = '';
r = f +
a
.map((v, i, a) => {
return codes[v];
})
.filter((v, i, a) => {
return ((i === 0) ? v !== codes[f] : v !== a[i - 1]);
})
.join('');
return (`${r }000`).slice(0, 4).toUpperCase();
};
}
module.exports = Scraper;

30
ncas/at.js
View File

@ -89,9 +89,9 @@ class ATScrape extends Scraper {
details['permissions'] = [];
const permissionsDiv = $('div.modal-body');
$(permissionsDiv).find('h4').each((i, item) => {
const code = this._cleanUp($(item).text());
const description = this._cleanUp($(item).next().text());
details['permissions'].push({ 'code': code, 'description': description });
const heading = this._cleanUp($(item).text());
const body = $(item).next().html().split('<br>').map(x => this._cleanUp(x)).filter(x => x != "");
details['permissions'].push({ heading, body });
});
}
@ -164,21 +164,19 @@ class ATScrape extends Scraper {
const entities = $('div.company-details-wrap');
const href = await this.page.url();
entities.each(async (i, item) => {
const noWhiteSpace = /\W/g;
const details = this.extractEntityDetails($(item).html());
const id = this._makeFieldName(details.name);
const entity = removeAccents.remove(id.trim());
const filename = [this.modePrefix[this.mode], entity.replace(noWhiteSpace, '_')].join('');
const filename = [this.modePrefix[this.mode], entity.replace(noWhiteSpace, '_'), '.json'].join('');
const filePath = `${this.path}/${filename}`.substring(0, 240);
jsonfile.writeFile(`${filePath}.json`, { details });
jsonfile.writeFile(`${filePath}`, { details });
this.getCurrentMode().links.push({
'id': id,
'href': await this.page.url(),
'filename': filename
});
this.getCurrentMode().links.push({ id, href, filename });
});
logger.info(`${entities.length} ${this.modeNames[this.mode]} entities scraped.`);
@ -213,7 +211,6 @@ class ATScrape extends Scraper {
await this.entityResultsPageProcessor();
else
logger.error(`Page url not recognised: ${pageUrl.href}`);
}
getCurrentMode() {
@ -234,12 +231,11 @@ class ATScrape extends Scraper {
getNextUrl() {
if (this.getCurrentMode().urlStep < this.getCurrentMode().urls.length - 1)
this.getCurrentMode().urlStep++;
else {
if (this.mode < this.modeNames.length - 1)
this.mode++;
else
return null;
}
else
if (this.mode < this.modeNames.length - 1)
this.mode++;
else
return null;
return this.getCurrentMode().urls[this.getCurrentMode().urlStep];
}

4
ncas/be.js
View File

@ -268,9 +268,9 @@ class BEScrape extends Scraper {
const id = this.getIdByEntityName(entity.name);
// create json file for each entity
const filename = [this.modePrefix[this.mode], id].join('');
const filename = [this.modePrefix[this.mode], id, '.json'].join('');
const filePath = `${this.path}/${filename}`.substring(0, 240);
jsonfile.writeFile(`${filePath}.json`, { 'details': entity , metadataFileName});
jsonfile.writeFile(filePath, { 'details': entity , metadataFileName});
// add entity details to "links" so that index file can be generated later
this.getCurrentMode().links.push({

2
ncas/cy.js
View File

@ -15,6 +15,8 @@ class CYScrape extends Scraper {
super();
this.setID('CY');
this.addToBlockFilters(['recaptcha']);
this.on('done', () => {
this._done();
});

49
ncas/cz.js
View File

@ -11,7 +11,7 @@ class CZScrape extends Scraper {
constructor() {
super();
this.id = 'CZ';
this.setID('CZ');
this.version = '0.0.1-3';
this.captchas = ['iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAj0lEQVRYR+2XQQ6AIAwE7f8fXeOBBIlll8bYSNZzhemUZMHc3Y/CzwSQMWBm06GtTDU1ghIAtGmkBNmgDZQBZDcejUQmoIEIYKb26Z/XANBMW+cjhABkQAZkQAZkYB8DLe0+i+PVeL3q+yhG8Q0vJBEA+5bZB6DvGN0TUde3tX75MGHnz9TRh5BZLFNTDnACDZUAsJw5oEAAAAAASUVORK5CYII=',
@ -218,23 +218,27 @@ class CZScrape extends Scraper {
*/
async entityCompleter(serviceObject) {
let cbFlag = false;
try{
if (serviceObject.current.authLink !== '' && !serviceObject.current.authProcess) {
await this._randomWait(this.page, 3, 5, 'Get Authorisations');
if (serviceObject.current.authLink !== '' && !serviceObject.current.authProcess) {
await this._randomWait(this.page, 3, 5, 'Get Authorisations');
await this._goto(serviceObject.current.authLink, { 'waitUntil':'networkidle0' });
await this._goto(serviceObject.current.authLink, { 'waitUntil':'networkidle2' });
return null;
}
return null;
if (typeof serviceObject.current.crossBorderLinks !== 'undefined' && !serviceObject.current.cbProcess && serviceObject.current.crossBorderLinks.length > 0) {
await this._randomWait(this.page, 3, 5, 'Get CBs');
// logger.info(`Crossborder for ${serviceObject.current.crossBorderLinks[serviceObject.current.crossBorderStep].name}`);
await this._goto(serviceObject.current.crossBorderLinks[serviceObject.current.crossBorderStep].href, { 'waitUntil':'networkidle0' });
}
else
cbFlag = true;
}
if (typeof serviceObject.current.crossBorderLinks !== 'undefined' && !serviceObject.current.cbProcess && serviceObject.current.crossBorderLinks.length > 0) {
await this._randomWait(this.page, 3, 5, 'Get CBs');
// logger.info(`Crossborder for ${serviceObject.current.crossBorderLinks[serviceObject.current.crossBorderStep].name}`);
await this._goto(serviceObject.current.crossBorderLinks[serviceObject.current.crossBorderStep].href, { 'waitUntil':'networkidle2' });
catch( err) {
logger.error(err);
}
else
cbFlag = true;
if( cbFlag === true) {
const filename = serviceObject.links[serviceObject.step].fileName;
@ -254,7 +258,7 @@ class CZScrape extends Scraper {
if (serviceObject.step < serviceObject.items) {
serviceObject.current = {};
await this._goto(serviceObject.links[serviceObject.step].href, { 'waitUntil':'networkidle2' });
await this._goto(serviceObject.links[serviceObject.step].href, { 'waitUntil':'networkidle0' });
}
else
this.emit('serviceDone');
@ -635,7 +639,7 @@ class CZScrape extends Scraper {
await this._randomWait(this.page, 3, 5, 'First sub section');
await this._goto(serviceObject.sectionLinks[serviceObject.indexStep], { 'waitUntil':'networkidle2' });
await this._goto(serviceObject.sectionLinks[serviceObject.indexStep], { 'waitUntil':'networkidle0' });
}
/**
@ -704,15 +708,16 @@ class CZScrape extends Scraper {
if (serviceObject.indexStep >= serviceObject.sectionLinks.length) {
this.inProgress = true;
serviceObject.items = serviceObject.links.length;
await this._randomWait(this.page, 3, 5, 'First page');
logger.info('goto', serviceObject.links[serviceObject.step].href);
await this._goto(serviceObject.links[serviceObject.step].href, { 'waitUntil':'networkidle2' });
await this._goto(serviceObject.links[serviceObject.step].href, { 'waitUntil':'networkidle0' });
}
else {
await this._randomWait(this.page, 3, 5, 'Next sub section');
await this._goto(serviceObject.sectionLinks[serviceObject.indexStep], { 'waitUntil':'networkidle2', 'timeout': 0 });
await this._goto(serviceObject.sectionLinks[serviceObject.indexStep], { 'waitUntil':'networkidle0', 'timeout': 5000 });
}
}
@ -837,12 +842,13 @@ class CZScrape extends Scraper {
*/
async processNewPage() {
// give the page a few seconds to settle
const errorPages = ['https://apl.cnb.cz/apljerrsdad/undefined', 'chrome-error://chromewebdata/'];
await this._randomWait(this.page, 3, 5);
const pageUrl = url.parse(await this.page.url());
if (pageUrl.href === 'chrome-error://chromewebdata/') {
logger.warn('Directed to: chrome-error://chromewebdata/');
if (errorPages.indexOf(pageUrl.href) !== -1) {
logger.warn(`Directed to: ${pageUrl.href}`);
this.emit('recover');
return;
@ -880,7 +886,8 @@ class CZScrape extends Scraper {
default:
if (process.env.NODE_ENV) {
await this._uploadError();
throw new Error(`Unknown page: ${pageUrl}`);
// throw new Error(`Unknown page: ${pageUrl}`);
this.emit('recover');
}
else {
logger.warn('processNewPage Fell through');
@ -1044,7 +1051,7 @@ class CZScrape extends Scraper {
//
await this.page.setViewport({ 'width': 1200, 'height': 800 });
await this._goto(this.startPage, { 'waitUntil':'networkidle2' });
await this._goto(this.startPage, { 'waitUntil':'networkidle0' });
await this._randomWait(this.page, 3, 5);
}

314
ncas/de.js
View File

@ -5,7 +5,7 @@ const cheerio = require('cheerio');
const path = require('path');
const jsonfile = require('jsonfile');
const removeAccents = require('remove-accents-diacritics');
const logger = require('log4js').getLogger('DE');
const logger = require('log4js').getLogger('(DE)');
const url = require('url');
logger.level = process.env.LOGGER_LEVEL || 'warn';
@ -123,94 +123,106 @@ class DEScrape extends Scraper {
}
async processCreditInstIndexPage() {
const noWhiteSpace = /\W/g;
logger.info('Building CI sub-index...');
try{
const noWhiteSpace = /\W/g;
logger.info('Building CI sub-index...');
const wantedRowType = ['CRR-Kreditinstitut'];
const currentPage = await this.page.evaluate(() => document);
const body = await this.page.content();
const $ = cheerio.load(body);
const wantedRowType = ['CRR-Kreditinstitut'];
const currentPage = await this.page.evaluate(() => document);
const body = await this.page.content();
const $ = cheerio.load(body);
const search = currentPage.location.search;
const params = this._getParamsFromUrl(search);
const search = currentPage.location.search;
const params = this._getParamsFromUrl(search);
const currentPageID = params['d-4012550-p'] || '';
const currentPageID = params['d-4012550-p'] || '';
await this._makeScreenshotV2(this.page, `${this.path}/credit_instititute_menu_${currentPageID}`, null);
await this._makeScreenshotV2(this.page, `${this.path}/credit_instititute_menu_${currentPageID}`, null);
await this._randomWait(this.page, 7, 10);
await this._randomWait(this.page, 7, 10);
const rows = $('#institut tr');
const rows = $('#institut tr');
rows.each((i, elm) => {
const rowClass = cheerio(elm).attr('class');
rows.each((i, elm) => {
const rowClass = cheerio(elm).attr('class');
if (typeof(rowClass) !== 'undefined') {
const children = cheerio(elm).children();
if (typeof(rowClass) !== 'undefined') {
const children = cheerio(elm).children();
const rowType = children.eq(1).text();
const rowType = children.eq(1).text();
if (wantedRowType.indexOf(rowType) !== -1) {
const name = this._cleanUp(children.eq(0).text());
const id = this._makeFieldName(name);
let href = cheerio(children.eq(0)).find('a').attr('href');
const params = this._getParamsFromUrl(href);
href = href.concat('&locale=en_GB');
if (wantedRowType.indexOf(rowType) !== -1) {
const name = this._cleanUp(children.eq(0).text());
const id = this._makeFieldName(name);
let href = cheerio(children.eq(0)).find('a').attr('href');
const params = this._getParamsFromUrl(href);
href = href.concat('&locale=en_GB');
// this is the one we want.
// this is the one we want.
this.creditServices.links.push({ name, id, href, params });
this.creditServices.links.push({ name, id, href, params });
}
}
});
const clicked = await this._findAndClick('.pagelinks a', 'Next');
if (!clicked) {
// come to the end of the index..
this.creditServices.done = true;
this.creditServices.items = this.creditServices.links.length;
this.emit('ciindexdone');
}
});
const clicked = await this._findAndClick('.pagelinks a', 'Next');
if (!clicked) {
// come to the end of the index..
this.creditServices.done = true;
this.creditServices.items = this.creditServices.links.length;
this.emit('ciindexdone');
}
catch( err) {
logger.error(err);
this.emit('recover');
}
}
async processCreditInstPage() {
const noWhiteSpace = /\W/g;
try{
const noWhiteSpace = /\W/g;
const id = this.creditServices.links[this.creditServices.step].id;
const name = this.creditServices.links[this.creditServices.step].name;
logger.info(`Process Credit Service entity ${this.creditServices.step} of ${this.creditServices.items} // ${name}`);
const id = this.creditServices.links[this.creditServices.step].id;
const name = this.creditServices.links[this.creditServices.step].name;
logger.info(`Process Credit Service entity ${this.creditServices.step} of ${this.creditServices.items} // ${name}`);
await this._randomWait(this.page, 3, 5);
await this._randomWait(this.page, 3, 5);
const body = await this.page.content();
const body = await this.page.content();
const details = await this.extractPaymentEntity(body);
const details = await this.extractPaymentEntity(body);
const entity = removeAccents.remove(details.description[0].trim());
const entity = removeAccents.remove(details.description[0].trim());
const filename = id.indexOf('?id=') === 0 ? this._makeFileName(entity) : this._makeFileName(id);
const filename = id.indexOf('?id=') === 0 ? this._makeFileName(entity) : this._makeFileName(id);
logger.debug('filename', filename);
logger.debug('filename', filename);
const filePath = `${this.path}/${filename}`.substring(0, 240);
const filePath = `${this.path}/${filename}`.substring(0, 240);
await this._makeScreenshotV2(this.page, `${filePath}_main`, null);
await this._makeScreenshotV2(this.page, `${filePath}_main`, null);
jsonfile.writeFileSync(`${filePath}.json`, details);
jsonfile.writeFileSync(`${filePath}.json`, details);
this.creditServices.links[this.creditServices.step].filename = `${filename}.json`;
this.creditServices.links[this.creditServices.step].filePath = `${filePath}`;
this.creditServices.step++;
this.creditServices.links[this.creditServices.step].filename = `${filename}.json`;
this.creditServices.links[this.creditServices.step].filePath = `${filePath}`;
this.creditServices.step++;
if (this.creditServices.step < this.creditServices.items) {
const newUrl = `https://portal.mvp.bafin.de/database/InstInfo/${this.creditServices.links[this.creditServices.step].href}`;
if (this.creditServices.step < this.creditServices.items) {
const newUrl = `https://portal.mvp.bafin.de/database/InstInfo/${this.creditServices.links[this.creditServices.step].href}`;
await this._goto(newUrl);
await this._goto(newUrl);
}
else
this.emit('creditinstdone');
}
catch( err) {
logger.error(err);
this.emit('recover');
}
else
this.emit('creditinstdone');
}
/**
@ -232,58 +244,64 @@ class DEScrape extends Scraper {
* @returns {Promise<{description: T[] | jQuery, permissions: {original: Array, translated: Array}}>}
*/
async extractPaymentEntity(html) {
const permissions = { 'original':[], 'translated':[] };
try{
const permissions = { 'original':[], 'translated':[] };
const newLine = /\n/g;
const $ = cheerio.load(html);
const newLine = /\n/g;
const $ = cheerio.load(html);
let description = $('#content > p').text().split(newLine).filter(line => line.length > 0);
let description = $('#content > p').text().split(newLine).filter(line => line.length > 0);
description = description.map((i) => {
return this._cleanUp(i.replace(/\t/g, '')).trim();
});
description = description.map((i) => {
return this._cleanUp(i.replace(/\t/g, '')).trim();
});
description = description.filter(item => item.length > 0);
description = description.filter(item => item.length > 0);
const rows = $('#erlaubnis > tbody tr');
const rows = $('#erlaubnis > tbody tr');
rows.each((index, item) => {
const cells = $(item).find('td');
rows.each((index, item) => {
const cells = $(item).find('td');
const service = $(cells.get(0)).text();
const startAuth = $(cells.get(1)).text();
const endAuth = $(cells.get(2)).text();
const service = $(cells.get(0)).text();
const startAuth = $(cells.get(1)).text();
const endAuth = $(cells.get(2)).text();
const reason = (cells.length === 4) ? $(cells.get(3)).text() : '';
const reason = (cells.length === 4) ? $(cells.get(3)).text() : '';
const phrasing = service.split(' (§');
const translated = this._translate(phrasing[0]);
const phrasing = service.split(' (§');
const translated = this._translate(phrasing[0]);
phrasing[0] = (translated !== '') ? translated : phrasing[0];
phrasing[0] = (translated !== '') ? translated : phrasing[0];
const newObjTrans = {
'service': phrasing.join(' (§'),
startAuth,
endAuth
};
const newObjTrans = {
'service': phrasing.join(' (§'),
startAuth,
endAuth
};
const newObj = {
service,
startAuth,
endAuth
};
const newObj = {
service,
startAuth,
endAuth
};
if (cells.length === 4) {
newObj.reason = reason;
newObjTrans.reason = reason;
}
if (cells.length === 4) {
newObj.reason = reason;
newObjTrans.reason = reason;
}
permissions.translated.push(newObjTrans);
permissions.translated.push(newObjTrans);
permissions.original.push(newObj);
});
permissions.original.push(newObj);
});
return { description, permissions };
return { description, permissions };
}
catch( err) {
logger.error(err);
this.emit('recover');
}
}
/**
@ -291,58 +309,64 @@ class DEScrape extends Scraper {
* @returns {Promise<void>}
*/
async processEntity() {
const noWhiteSpace = /\W/g;
if (!this.subIndex.done) {
// We should not be here quite yet, so add this to subindex;
const currentPage = await this.page.evaluate(() => document);
try{
const noWhiteSpace = /\W/g;
if (!this.subIndex.done) {
// We should not be here quite yet, so add this to subindex;
const currentPage = await this.page.evaluate(() => document);
const location = currentPage.location;
const id = location.search;
let href = location.href;
href = href.concat('&locale=en_GB');
const location = currentPage.location;
const id = location.search;
let href = location.href;
href = href.concat('&locale=en_GB');
this.paymentServices.links.push({ id, href });
this.paymentServices.links.push({ id, href });
this.index.step++;
this.index.step++;
if (this.index.step < this.index.items)
this.emit('nextsubindex');
if (this.index.step < this.index.items)
this.emit('nextsubindex');
else {
logger.info('Sub indexing done...');
this.subIndex.done = true;
this.paymentServices.items = this.paymentServices.links.length;
this.emit('subindexdone');
}
}
else {
logger.info('Sub indexing done...');
this.subIndex.done = true;
this.paymentServices.items = this.paymentServices.links.length;
this.emit('subindexdone');
const id = this.paymentServices.links[this.paymentServices.step].id;
// logger.info('Process entity:', id);
logger.info(`Process entity ${this.paymentServices.step} of ${this.paymentServices.items} // ${id}`);
await this._randomWait(this.page, 3, 5);
const body = await this.page.evaluate(() => document.documentElement.outerHTML);
const details = await this.extractPaymentEntity(body);
const entity = removeAccents.remove(details.description[0].trim());
// const filename = id.indexOf('?id=') === 0 ? `ps_${entity.replace(noWhiteSpace, '_')}` : `ps_${id.replace(noWhiteSpace, '_')}`;
const filename = id.indexOf('?id=') === 0 ? this._makeFileName(entity) : this._makeFileName(id);
logger.debug('filename', filename);
await this._makeScreenshotV2(this.page, `${this.path}/${filename}_main`, null);
jsonfile.writeFileSync(`${this.path}/${filename}.json`, details);
this.paymentServices.links[this.paymentServices.step].filename = `${filename}.json`;
this.paymentServices.step++;
if (this.paymentServices.step < this.paymentServices.items)
await this._goto(this.paymentServices.links[this.paymentServices.step].href);
else
this.emit('processdone');
}
}
else {
const id = this.paymentServices.links[this.paymentServices.step].id;
// logger.info('Process entity:', id);
logger.info(`Process entity ${this.paymentServices.step} of ${this.paymentServices.items} // ${id}`);
await this._randomWait(this.page, 3, 5);
const body = await this.page.evaluate(() => document.documentElement.outerHTML);
const details = await this.extractPaymentEntity(body);
const entity = removeAccents.remove(details.description[0].trim());
// const filename = id.indexOf('?id=') === 0 ? `ps_${entity.replace(noWhiteSpace, '_')}` : `ps_${id.replace(noWhiteSpace, '_')}`;
const filename = id.indexOf('?id=') === 0 ? this._makeFileName(entity) : this._makeFileName(id);
logger.debug('filename', filename);
await this._makeScreenshotV2(this.page, `${this.path}/${filename}_main`, null);
jsonfile.writeFileSync(`${this.path}/${filename}.json`, details);
this.paymentServices.links[this.paymentServices.step].filename = `${filename}.json`;
this.paymentServices.step++;
if (this.paymentServices.step < this.paymentServices.items)
await this._goto(this.paymentServices.links[this.paymentServices.step].href);
else
this.emit('processdone');
catch( err) {
logger.error(err);
this.emit('reover');
}
}
@ -453,12 +477,12 @@ class DEScrape extends Scraper {
* @returns {Promise<void>}
*/
async attachEvents() {
this.on('startcredit', async function() {
this.on('startcredit', async () => {
logger.info('Starting Credit Institutes');
await this._goto(this.credit);
});
this.on('processdone', async function() {
this.on('processdone', async () => {
logger.warn('Payment Entities done', this.paymentServices.items);
jsonfile.writeFileSync(`${this.path}/paymentServices.json`, { 'links': this.paymentServices.links });
@ -469,19 +493,19 @@ class DEScrape extends Scraper {
await this._goto(this.emoneyUrl);
});
this.on('subindexdone', async function() {
this.on('subindexdone', async () => {
logger.info('Sub Index done', this.paymentServices.items);
logger.info(this.paymentServices.links[this.paymentServices.step].href);
await this._goto(this.paymentServices.links[this.paymentServices.step].href);
});
this.on('indexdone', async function() {
this.on('indexdone', async () => {
logger.info('Index done', this.index.items);
logger.info(this.index.links[this.index.step].href);
await this._goto(this.index.links[this.index.step].href);
});
this.on('ciindexdone', async function() {
this.on('ciindexdone', async () => {
logger.info('CI Index done', this.creditServices.items);
logger.info(this.creditServices.links[this.creditServices.step].href);
@ -489,7 +513,7 @@ class DEScrape extends Scraper {
await this._goto(newUrl);
});
this.on('creditinstdone', async function() {
this.on('creditinstdone', async () => {
logger.debug('Credit Institutes done', this.paymentServices.items);
jsonfile.writeFileSync(`${this.path}/creditServices.json`, { 'links':this.creditServices.links });
@ -499,7 +523,7 @@ class DEScrape extends Scraper {
this.emit('done');
});
this.on('nextsubindex', async function() {
this.on('nextsubindex', async () => {
logger.debug(this.index.links[this.index.step].href);
await this._goto(this.index.links[this.index.step].href);
});

163
ncas/dkV2.js
View File

@ -1,6 +1,6 @@
const Scraper = require('../helpers/scraper');
const path = require('path');
const logger = require('log4js').getLogger('DK');
const logger = require('log4js').getLogger('(DK)');
const url = require('url');
logger.level = process.env.LOGGER_LEVEL || 'warn';
@ -9,7 +9,7 @@ class DKScrape extends Scraper {
constructor(checkForLock = true) {
super();
this.id = 'DK';
this.setID('DK');
this.on('done', () => {
this._done();
@ -55,14 +55,20 @@ class DKScrape extends Scraper {
* @returns {Promise<void>}
*/
async renameFile() {
const filename = this.filenames[this.step];
if (!this.errored) {
const filename = this.filenames[this.step];
const sourceFile = 'Finanstilsynets virksomhedsregister - SQL.xlsx';
const sourceFile = 'Finanstilsynets virksomhedsregister - SQL.xlsx';
const origFile = `${this.path}/${sourceFile}`;
const newFile = `${this.path}/${filename}.xlsx`;
const origFile = `${this.path}/${sourceFile}`;
const newFile = `${this.path}/${filename}.xlsx`;
await this._renameFile(origFile, newFile);
await this._renameFile(origFile, newFile);
await this._randomWait(this.page, 5, 7, 'after renameFile');
}
else
logger.warn('Skipping renameFile');
}
/**
@ -70,17 +76,26 @@ class DKScrape extends Scraper {
* @returns {Promise<void>}
*/
async clickReturn() {
logger.debug('clickReturn');
await this.iframe.waitForSelector('#lsAnalysisPage > div > div:nth-child(2)', { 'visible':true, 'timeout':75000 }).then(async (elm) => {
console.log('found');
await elm.click({ 'delay':90 });
}).catch((e) => {
logger.error('iframe missing stuff', e);
// pageLoaded = false;
});
if (!this.errored) {
logger.debug('clickReturn');
await this.iframe.waitForSelector('#lsAnalysisPage > div > div:nth-child(2)', { 'visible':true, 'timeout':75000 }).then(async (elm) => {
console.log('found');
await elm.click({ 'delay':90 });
await this._randomWait(this.page, 5, 7, 'after clickReturn click');
this.step++;
}).catch((e) => {
logger.error('iframe missing stuff (clickReturn)', e);
// pageLoaded = false;
this.emit('recover');
});
}
else
logger.warn('Skipping clickReturn');
await this._randomWait(this.page, 2, 3, 'after clickReturn click');
/*
await this._randomWait(this.page, 5, 7, 'after clickReturn click');
this.step++;
*/
}
/**
@ -88,22 +103,26 @@ class DKScrape extends Scraper {
* @returns {Promise<void>}
*/
async scrollContainer() {
await this.page.evaluate(() => {
console.log('window.innerWidth', window.innerWidth);
window.scrollBy(window.innerWidth, window.innerHeight);
});
if (!this.errored) {
await this.page.evaluate(() => {
console.log('window.innerWidth', window.innerWidth);
window.scrollBy(window.innerWidth, window.innerHeight);
});
await this._randomWait(this.page, 2, 2, 'scroll x?');
await this._randomWait(this.page, 2, 2, 'scroll x?');
this.page.mouse.move(1061, 437);
await this._randomWait(this.page, 2, 3, 'bottom right scroll arrow');
this.page.mouse.move(1061, 437);
await this._randomWait(this.page, 2, 3, 'bottom right scroll arrow');
for(let count = 0; count < 15; count++) {
this.page.mouse.click(1061, 437, { 'delay':500 });
await this._randomWait(this.page, 1, 2, 'scrolling');
for(let count = 0; count < 15; count++) {
this.page.mouse.click(1061, 437, { 'delay':821 });
await this._randomWait(this.page, 4, 7, 'scrolling');
}
await this._randomWait(this.page, 5, 7, 'after scroll');
}
await this._randomWait(this.page, 4, 5, 'after scroll');
else
logger.warn('Skipping scrollContainer');
}
/**
@ -111,38 +130,43 @@ class DKScrape extends Scraper {
* @returns {Promise<void>}
*/
async clickExport() {
logger.debug('clickExport');
if (!this.errored) {
logger.debug('clickExport');
await this.movePageToTop();
await this.movePageToTop();
await this._randomWait(this.page, 2, 2, 'Move to top');
await this._randomWait(this.page, 2, 2, 'Move to top');
const filename = this.filenames[this.step];
const filename = this.filenames[this.step];
const filePath = `${this.path}/${filename}`.substring(0, 240);
const filePath = `${this.path}/${filename}`.substring(0, 240);
await this._makeScreenshotV2(this.page, `${filePath}_main`, null);
await this._makeScreenshotV2(this.page, `${filePath}_main`, null);
await this._randomWait(this.page, 4, 4, 'Screenshot');
await this._randomWait(this.page, 4, 4, 'Screenshot');
this.page.mouse.move(175, 440);
await this._randomWait(this.page, 2, 3, 'Move 175, 440');
this.page.mouse.move(175, 440);
await this._randomWait(this.page, 2, 3, 'Move 175, 440');
this.page.mouse.click(175, 440, { 'button':'right', 'delay':90 });
this.page.mouse.click(175, 440, { 'button':'right', 'delay':90 });
await this._randomWait(this.page, 2, 3, 'Click 175, 440');
await this._randomWait(this.page, 2, 3, 'Click 175, 440');
await this.page._client.send('Page.setDownloadBehavior', { 'behavior': 'allow', 'downloadPath': this.path });
await this.page._client.send('Page.setDownloadBehavior', { 'behavior': 'allow', 'downloadPath': this.path });
await this.iframe.waitForSelector('div.lsDialogContent > div:nth-child(2)', { 'visible':true, 'timeout':75000 }).then(async (elm) => {
console.log('found');
await elm.click({ 'delay':90 });
}).catch((e) => {
logger.error('iframe missing stuff', e);
// pageLoaded = false;
});
await this._randomWait(this.page, 2, 3, 'after clickExport click');
await this.iframe.waitForSelector('div.lsDialogContent > div:nth-child(2)', { 'visible':true, 'timeout':75000 }).then(async (elm) => {
console.log('found');
await elm.click({ 'delay':90 });
await this._randomWait(this.page, 5, 7, 'after clickExport click');
}).catch((e) => {
logger.error('iframe missing stuff (clickExport)', e);
this.errored = true;
this.emit('recover');
// pageLoaded = false;
});
}
else
logger.warn('Skipping clickExport');
}
/**
@ -159,12 +183,13 @@ class DKScrape extends Scraper {
await this.iframe.waitForSelector('#lsAnalysisPage > div > div:nth-child(11)', { 'visible':true, 'timeout':75000 }).then(async (elm) => {
console.log('found');
await elm.click({ 'delay':90 });
await this._randomWait(this.page, 5, 7, 'after clickSearch click');
}).catch((e) => {
logger.error('iframe missing stuff', e);
// pageLoaded = false;
this.emit('recover');
this.errored = true;
});
await this._randomWait(this.page, 2, 3, 'after clickSearch click');
}
/**
@ -191,7 +216,7 @@ class DKScrape extends Scraper {
this.page.mouse.click(400, 434);
await this._randomWait(this.page, 2, 3, 'Click 400, 434');
await this._randomWait(this.page, 5, 7, 'Click 400, 434');
}
/**
@ -219,7 +244,7 @@ class DKScrape extends Scraper {
this.page.mouse.click(400, 585);
await this._randomWait(this.page, 2, 3, 'Click 400, 585');
await this._randomWait(this.page, 5, 7, 'Click 400, 585');
}
/**
@ -246,7 +271,7 @@ class DKScrape extends Scraper {
this.page.mouse.click(400, 473);
await this._randomWait(this.page, 2, 3, 'Click 400, 473');
await this._randomWait(this.page, 5, 7, 'Click 400, 473');
}
/**
@ -273,7 +298,7 @@ class DKScrape extends Scraper {
this.page.mouse.click(400, 631);
await this._randomWait(this.page, 2, 3, 'Click 400, 631');
await this._randomWait(this.page, 5, 7, 'Click 400, 631');
}
/**
@ -300,7 +325,7 @@ class DKScrape extends Scraper {
this.page.mouse.click(400, 473);
await this._randomWait(this.page, 2, 3, 'Click 400, 473');
await this._randomWait(this.page, 5, 7, 'Click 400, 473');
}
/**
@ -308,7 +333,7 @@ class DKScrape extends Scraper {
* @returns {Promise<void>}
*/
async motions() {
do
do
switch(this.step) {
case 0:
@ -392,8 +417,22 @@ class DKScrape extends Scraper {
break;
}
while(!this.complete );
while(!this.complete && !this.errored);
}
/**
*
* @param restartURL
* @returns {Promise<void>}
* @private
*/
async __recover() {
// need to reset the error status then restart
logger.info('OVERLOAD __recover');
this.errored = false;
super.__recover(this.startPage);
}
/**
@ -410,7 +449,7 @@ class DKScrape extends Scraper {
await this.motions();
}).catch((e) => {
logger.error('processEntityDetails', e);
logger.error('waitForIframe', e);
// pageLoaded = false;
});
}
@ -448,6 +487,8 @@ class DKScrape extends Scraper {
this.step = 0;
this.complete = false;
this.errored = false;
this.filenames = ['paymentServices01', 'paymentServices02', 'eMoney01', 'eMoney02', 'creditServices01', 'creditServices02'];
this.startPage = 'https://virksomhedsregister.finanstilsynet.dk/listeudtr%C3%A6k-en.html';
@ -458,7 +499,7 @@ class DKScrape extends Scraper {
logger.error(err);
});
await this._initBrowser();
await this._initBrowser(false);
await this._createBrowserPage();
await this._makeResponsive();

6
ncas/ee.js
View File

@ -2,7 +2,7 @@ const Scraper = require('../helpers/scraper');
const cheerio = require('cheerio');
const path = require('path');
const jsonfile = require('jsonfile');
const logger = require('log4js').getLogger('EE');
const logger = require('log4js').getLogger('(EE)');
const url = require('url');
const removeAccents = require('remove-accents-diacritics');
@ -12,7 +12,7 @@ class EEScrape extends Scraper {
constructor() {
super();
this.id = 'EE';
this.setID('EE');
this.on('done', () => {
this._done();
@ -574,7 +574,7 @@ class EEScrape extends Scraper {
if (this.paymentServices.indexStep >= this.paymentServices.urls.length) {
this.workMode = 1;
logger.debug(JSON.stringify(this.paymentServices));
// logger.debug(JSON.stringify(this.paymentServices));
newUrl = this.paymentServices.links[this.paymentServices.step].href;
}
else

17
ncas/es.js
View File

@ -1,7 +1,7 @@
const Scraper = require('../helpers/scraper');
const cheerio = require('cheerio');
const path = require('path');
const logger = require('log4js').getLogger('ES');
const logger = require('log4js').getLogger('(ES)');
const url = require('url');
const querystring = require('querystring');
const removeAccents = require('remove-accents-diacritics');
@ -13,7 +13,7 @@ class ESScrape extends Scraper {
constructor() {
super();
this.id = 'ES';
this.setID('ES');
this.on('done', () => {
this._done();
@ -279,7 +279,7 @@ class ESScrape extends Scraper {
const { name, id } = serviceObject.links[serviceObject.step];
logger.info(`Process ${this.modeTitles[this.mode]} entity ${serviceObject.step + 1} of ${serviceObject.items} // ${name}`);
await this.page.waitForSelector('td.tdContenido', { 'visible':true, 'timeout':7500 }); // Wait for buttons at bottom of table to be visible
await this.page.waitForSelector('td.tdContenido', { 'visible':true, 'timeout':90000 }); // Wait for buttons at bottom of table to be visible
await this._randomWait(this.page, 3, 5);
@ -361,6 +361,7 @@ class ESScrape extends Scraper {
await this.processRedirector();
else {
await this._uploadError();
this.emit('recover');
throw new Error(`Unknown page: ${pageUrl}`);
}
}
@ -370,7 +371,7 @@ class ESScrape extends Scraper {
* @returns {Promise<void>}
*/
async attachEvents() {
this.on('serviceDone', async function() {
this.on('serviceDone', async () => {
switch (this.mode) {
case 0:
@ -388,6 +389,8 @@ class ESScrape extends Scraper {
}
});
//
this.on('psindexdone', async () => {
this.paymentServices.items = this.paymentServices.links.length;
logger.info(`${this.paymentServices.items} paymentServices items indexed`);
@ -397,6 +400,8 @@ class ESScrape extends Scraper {
await this._goto(newUrl);
});
//
this.on('emindexdone', async () => {
this.emoneyServices.items = this.emoneyServices.links.length;
logger.info(`${this.emoneyServices.items} emoneyServices items indexed`);
@ -406,6 +411,8 @@ class ESScrape extends Scraper {
await this._goto(newUrl);
});
//
this.on('ciindexdone', async () => {
this.creditServices.items = this.creditServices.links.length;
logger.info(`${this.creditServices.items} creditServices items indexed`);
@ -415,6 +422,8 @@ class ESScrape extends Scraper {
await this._goto(newUrl);
});
//
this.on('indexdone', async () => {
switch (this.mode) {

231
ncas/it.js
View File

@ -2,7 +2,7 @@ const Scraper = require('../helpers/scraper');
const cheerio = require('cheerio');
const path = require('path');
const jsonfile = require('jsonfile');
const logger = require('log4js').getLogger('IT');
const logger = require('log4js').getLogger('(IT)');
const url = require('url');
logger.level = process.env.LOGGER_LEVEL || 'warn';
@ -56,16 +56,21 @@ class ITscrape extends Scraper {
* @returns {Promise<void>}
*/
async forceEnglish() {
await this._randomWait(this.page, 2, 2, 'Force English');
const localStorage = await this._getLocalStorage();
await this.page.waitForSelector('#bs-example-navbar-collapse-1 > ul > li.dropdown > a', { 'visible':true, 'timeout':7500 }).then(async (elm) => {
await elm.click({ 'delay':Scraper.notARobot() });
await this._randomWait(this.page, 2, 2);
}).catch(() => {
logger.debug('No Language button');
});
const userLang = localStorage['userLang'].trim();
await this._findAndClick('#bs-example-navbar-collapse-1 > ul > li.dropdown.open > ul > li:nth-child(2) > a');
if (userLang !== 'en') {
await this._randomWait(this.page, 2, 2, 'Force English');
await this.page.waitForSelector('#bs-example-navbar-collapse-1 > ul > li.dropdown > a', { 'visible':true, 'timeout':7500 }).then(async (elm) => {
await elm.click({ 'delay':Scraper.notARobot() });
await this._randomWait(this.page, 2, 2);
}).catch(() => {
logger.debug('No Language button');
});
await this._findAndClick('#bs-example-navbar-collapse-1 > ul > li.dropdown.open > ul > li:nth-child(2) > a');
}
}
/**
@ -74,6 +79,8 @@ class ITscrape extends Scraper {
*/
async handleFrontPage() {
let pageReturned = false;
this.processing = false;
await this._randomWait(this.page, 3, 5, 'handleFrontPage');
await this.page.waitFor('ul.linkgroup a', { 'visible':true }).then(async (elm) => {
@ -86,16 +93,16 @@ class ITscrape extends Scraper {
await this.page.waitFor('#my-container > div.container > div', { 'visible':true, 'timeout':7500 }).then(() => {
pageReturned = true;
}).catch(async () => {
logger.info('We didnt transition back correctly, forcing another click..\n');
logger.info('(handleFrontPage) We didnt transition back correctly, forcing another click..\n');
});
if (!pageReturned) {
await this.page.hover('ul.linkgroup a').catch((err) => {
logger.debug(err.name);
logger.debug(err.name, 'hover ul.linkgroup a');
});
await this.page.focus('ul.linkgroup a').catch((err) => {
logger.debug(err.name);
logger.debug(err.name, 'focus ul.linkgroup a');
});
await this.page.waitFor('ul.linkgroup a', { 'visible':true }).then(async (elm) => {
@ -106,7 +113,7 @@ class ITscrape extends Scraper {
}
}
while(!pageReturned);
while(!pageReturned && !this.processing);
// Supervisory registers and lists
}
@ -126,6 +133,11 @@ class ITscrape extends Scraper {
logger.warn('Ajax loading shroud not removed after 90 seconds');
});
await this.page.waitForSelector('div.loading', { 'visible':true, 'timeout':10000 }).catch((e) => {
logger.warn('Ajax loading shroud still there...');
});
// body > app-root > div:nth-child(2) > div
await this.page.waitForSelector('ul.nav.navbar-nav.navbar-center li a', { 'visible':false, 'timeout':90000 }).then(async (elm) => {
await elm.click({ 'delay':Scraper.notARobot() });
await this._randomWait(this.page, 5, 8, 'await transition');
@ -291,17 +303,18 @@ class ITscrape extends Scraper {
await this.page.waitForSelector('div.loading', { 'visible':false, 'timeout':25000 });
let btnSuccess = false;
let breakCount = 0;
do {
await this.page.waitForSelector('button.btn.btn-success', { 'visible':true, 'timeout':2500 }).then(async (elm) => {
await this.page.waitForSelector('button.btn.btn-success', { 'visible':true, 'timeout':45000 }).then(async (elm) => {
await elm.click({ 'delay':Scraper.notARobot() });
}).catch(() => {
btnSuccess = true;
});
await this._randomWait(this.page, 1, 1, 'preparePSSearch btnSuccess');
breakCount++;
}
while(!btnSuccess);
while(!btnSuccess && breakCount < 5);
this.page.waitFor('app-int-albi-grid-result').then(async () => {
//
@ -328,66 +341,30 @@ class ITscrape extends Scraper {
/**
*
* @returns {Promise<{registry, authority, registers}>}
* @param filePath
* @returns {Promise<void>}
*/
async processPSDetail() {
let registry = {}, registers = {}, authority = {};
async processPSDetailV2(filePath) {
await this._randomWait(this.page, 1, 3, 'processPSDetailV2: Screenshots');
await this._randomWait(this.page, 3, 3, 'processPSDetail: AJAX');
// await this._makeScreenshotV2(this.page, `${filePath}_main`, null);
await this.page.waitFor('#sub-navbar > app-int-albi > app-int-albi-details > div > div.card.card-title > span > span', { 'visible': true }).catch((err) => {
logger.warn('AJAX data has failed to load');
logger.debug(err);
return { registry, registers, authority };
});
await this.page.waitFor('app-int-albi-details').then(async () => {
await this.forceScrollToTop();
const body = await this.page.content();
registry = await this.extractPSRegistry(body);
await this._randomWait(this.page, 2, 2, 'processPSDetail app-int-albi-details');
}).catch(async (err) => {
if (process.env.NODE_ENV) {
await this._uploadError();
throw new Error('processPSDetail\n', err);
}
else
logger.error('processPSDetail\n', err);
});
await this._randomWait(this.page, 1, 1, 'processPSDetail after app-int-albi-details');
//
// wait for the data
await this.forceScrollToTop();
await this._makeScreenshotV2(this.page, `${filePath}_detail`, null);
// wait for Registers Tab
await this.page.waitFor('#sub-navbar > app-int-albi > app-int-albi-details > div > div:nth-child(2) > ul > li:nth-child(2) > a', { 'visible': true, 'timeout':10000 }).then(async (elm) => {
logger.debug('** Showing Registers Tab');
await this.page.waitFor('#sub-navbar > app-int-albi > app-int-albi-details > div > div:nth-child(2) > ul > li:nth-child(2) > a',
{ 'visible': true, 'timeout':10000 }).then(async (elm) => {
await elm.click({ 'delay':90 });
await this._randomWait(this.page, 1, 2, 'processPSDetailV2 wait for Tab trans');
await this.page.waitFor('app-details-albi', { 'visible': true, 'timeout':10000 }).then(async () => {
const body = await this.page.content();
registers = await this.extractPSRegisters(body);
await this._randomWait(this.page, 2, 2, 'processPSDetail app-details-albi');
}).catch(async (err) => {
if (process.env.NODE_ENV)
// await this._uploadError();
throw new Error('No tab transition\n', err);
else
logger.error('No tab transition');
await this._makeScreenshotV2(this.page, `${filePath}_reg`, null);
await this._randomWait(this.page, 3, 5, 'processPSDetailV2 ss');
}).catch(async () => {
logger.info('No tab transition');
});
await this._randomWait(this.page, 1, 1, 'processPSDetail after app-details-albi');
}).catch((err) => {
logger.warn('No "registers" Block...');
logger.debug(err);
@ -395,36 +372,29 @@ class ITscrape extends Scraper {
// wait for Activity Tab
await this.forceScrollToTop();
await this.page.waitFor('#sub-navbar > app-int-albi > app-int-albi-details > div > div:nth-child(2) > ul > li:nth-child(3) > a', { 'visible': true, 'timeout':10000 }).then(async (elm) => {
logger.debug('** Showing Activity Tab');
await this.page.waitFor('#sub-navbar > app-int-albi > app-int-albi-details > div > div:nth-child(2) > ul > li:nth-child(3) > a',
{ 'visible': true, 'timeout':10000 }).then(async (elm) => {
await elm.click({ 'delay':90 });
let pageReturned = false;
await this._randomWait(this.page, 1, 2, 'processPSDetailV2 wait for Tab B trans');
do
await this.page.waitFor('app-details-att-autorizzate', { 'visible': true, 'timeout':10000 }).then(async () => {
pageReturned = true;
const body = await this.page.content();
authority = await this.extractPSAuthority(body);
await this._randomWait(this.page, 2, 2, 'processPSDetail app-details-att-autorizzate');
}).catch(async (err) => {
// screen shot
await this._makeScreenshotV2(this.page, `${filePath}_activity`, null);
await this._randomWait(this.page, 3, 5, 'processPSDetailV2 ss');
}).catch(async () => {
await this.forceScrollToTop();
await this._findAndClick('#sub-navbar > app-int-albi > app-int-albi-details > div > div:nth-child(2) > ul > li:nth-child(3) > a');
if (process.env.NODE_ENV)
throw new Error('No tab transition\n', err);
else
logger.warn('No tab transition');
logger.info('No tab transition');
});
while(!pageReturned);
}).catch((err) => {
logger.warn('No "Activity" Block...');
logger.debug(err);
});
return { registry, registers, authority };
}
/**
@ -443,8 +413,9 @@ class ITscrape extends Scraper {
await this.page.waitFor('app-int-albi-grid-result').then(() => {
pageReturned = true;
}).catch(async (err) => {
logger.warn('We didnt transition back correctly, forcing another click..\n', err);
logger.warn('(returnToPSList) We didnt transition back correctly, forcing another click..\n', err);
await this._randomWait(this.page, 1, 1, 'one sec');
await this.forceScrollToTop();
await this._findAndClick('#sub-navbar > giava-breadcrumb > ol > li:nth-child(3) > a');
@ -482,7 +453,6 @@ class ITscrape extends Scraper {
divs.each((index, item) => {
const itemText = this._cleanUp($(item).text());
const itemName = $(item).attr('col-id');
// logger.info(`>> ${index}`, itemName, itemText);
entries[itemName] = itemText;
});
@ -521,6 +491,8 @@ class ITscrape extends Scraper {
let maxPages = 0;
let rowsInPass;
this.processing = true;
await this.psSetListCount(_defaultMaxPerPage);
const maxRows = await this.psGetMaxRows();
let remainingRows = maxRows;
@ -594,32 +566,38 @@ class ITscrape extends Scraper {
serviceObject.links.push({ uid, 'fileName':`${fileName}.json`, 'name':clickName });
// Go into the detail
await clickable.click();
await clickable.click().catch((e) => {
logger.debug('something happend with the click', e);
throw new Error(e);
});
await this._randomWait(this.page, 3, 4, 'processAGTableV3 before next');
remainingRows--;
await this.page.waitFor('app-int-albi-details').then(
await this.doAlbiDetails(filePath, newEntry)
).catch(async (err) => {
await this.page.waitFor('app-int-albi-details').catch(async (err) => {
logger.error('No detail transition', err);
this.emit('recover');
if (process.env.NODE_ENV)
await this._uploadError();
});
await this.doAlbiDetailsV2(filePath, newEntry).catch(async (err) => {
logger.error('doAlbiDetailsV2 had an issue', err);
this.emit('recover');
});
}
}
}
if (maxPages > 0) {
if (maxPages > 0 && !this.warning) {
logger.info('Clicking to the next page...');
const nextButton = await this.page.$$('#sub-navbar > app-int-albi > app-int-albi-grid-result > grid-pagination > div > div > div:nth-child(5) > button');
const buttonDisabled = await this.page.evaluate(el => el.disabled, nextButton[0]);
const buttonDisabled = await this.page.evaluate(el => el.disabled, nextButton[0]).catch((e) => {
logger.warn('Could not mark the button as disabled');
});
if (!buttonDisabled) {
this._findAndClick('#sub-navbar > app-int-albi > app-int-albi-grid-result > grid-pagination > div > div > div:nth-child(5) > button');
await this._randomWait(this.page, 5, 5, 'processAGTableV3 next page click');
@ -632,30 +610,64 @@ class ITscrape extends Scraper {
this.emit('doneProcessingGrid');
}
async doAlbiDetails(filePath, newEntry) {
/**
*
* @param filePath
* @param newEntry
* @returns {Promise<string>}
*/
async doAlbiDetailsV2(filePath, newEntry) {
const empty = { };
logger.debug('++ doAlbiDetailsV2');
let data;
try{
// process the page
const data = await this.processPSDetail();
data.details = newEntry;
logger.info(`Saving ${filePath}.json`);
await jsonfile.writeFile(`${filePath}.json`, data);
let breaker = 0;
await this._randomWait(this.page, 5, 7, 'doAlbiDetails');
do{
await this._randomWait(this.page, 1, 3, 'Retrieve from localStorage');
const localStorage = await this._getLocalStorage();
// Retun back to list
logger.debug('>> typeof(localStorage[\'intermediaryDetails\'])', typeof(localStorage['intermediaryDetails']));
await this.returnToPSList();
if (typeof(localStorage['intermediaryDetails']) !== 'undefined' )
data = JSON.parse(localStorage['intermediaryDetails']) ;
else
await this._randomWait(this.page, 2, 2, 'wait for the localStorage load');
logger.debug('>> typeof(data)', typeof(data));
breaker++;
}
while((typeof(data) === 'undefined' || data === null) && breaker < 10);
if (typeof(data) === 'undefined' || JSON.stringify(empty) === JSON.stringify(data)) {
logger.warn('localStorage.intermediaryDetails did not load');
throw new Error('localStorage.intermediaryDetails did not load');
}
else {
await this.processPSDetailV2(filePath);
logger.info(`Saving ${filePath}.json`);
await jsonfile.writeFile(`${filePath}.json`, data);
await this._randomWait(this.page, 5, 7, 'doAlbiDetailsV2');
// Retun back to list
await this.returnToPSList();
await this._randomWait(this.page, 2, 2, 'doAlbiDetailsV2 after returnToPSList');
}
await this._randomWait(this.page, 2, 2, 'doAlbiDetails after returnToPSList');
// wArray.push([uid, clickName]);
}
catch (err) {
logger.error('doAlbiDetails\n', err);
logger.error('doAlbiDetailsV2\n', err);
logger.debug('Converted data:', data);
this.emit('recover');
if (process.env.NODE_ENV)
await this._uploadError();
}
}
@ -719,9 +731,11 @@ class ITscrape extends Scraper {
}, 2500));
this.on('recover', this._debounce(async () => {
this.warning = true;
clearTimeout(this.backOffTimer);
logger.warn('Backing off for 5 minutes..');
this.processing = false;
const timeout = (60 * 1000) * 5;
this.backOffTimer = setTimeout(() => {
@ -731,6 +745,7 @@ class ITscrape extends Scraper {
}, 30000));
this.on('restart', this._debounce(async() => {
this.warning = false;
clearTimeout(this.backOffTimer);
logger.warn('Restarting::');
@ -804,7 +819,9 @@ class ITscrape extends Scraper {
async start() {
super._start();
try {
this.mode = 0;
this.mode = 2;
this.processing = false;
this.warning = false;
this.modeTitles = ['Payment Service', 'EMoney', 'Credit Services'];
@ -846,7 +863,7 @@ class ITscrape extends Scraper {
'workingIndex': 0
};
this.startPage = this.paymentServices.urls[0];
this.startPage = this.creditServices.urls[0];
this.emoneyUrl = '';
this.credit = '';
@ -858,7 +875,7 @@ class ITscrape extends Scraper {
logger.warn(err);
});
await this._initBrowser(true);
await this._initBrowser();
await this._createBrowserPage();
this.page.on('domcontentloaded', this._throttle(async () => {

1
ncas/lu.js
View File

@ -159,6 +159,7 @@ class LUScrape extends Scraper {
}
catch( err) {
logger.error(err);
this.emit('recover');
}
}

14
ncas/lv.js
View File

@ -2,7 +2,7 @@ const Scraper = require('../helpers/scraper');
const cheerio = require('cheerio');
const path = require('path');
const jsonfile = require('jsonfile');
const logger = require('log4js').getLogger('LV');
const logger = require('log4js').getLogger('(LV)');
const url = require('url');
const removeAccents = require('remove-accents-diacritics');
@ -12,7 +12,7 @@ class LVScrape extends Scraper {
constructor() {
super();
this.id = 'LV';
this.setID('LV');
this.on('done', () => {
this._done();
@ -57,7 +57,7 @@ class LVScrape extends Scraper {
const filename = this.modeNames[this.mode];
this._makeScreenshotV2(this.page, `${this.path}/${filename}_main_${serviceObject.indexStep}`, null);
this._makeScreenshotV2(this.page, `${this.path}/${filename}_main_${serviceObject.indexStep}`, 1500);
this.emit('indexdone');
}
@ -264,7 +264,7 @@ class LVScrape extends Scraper {
await this._randomWait(this.page, 3, 5);
await this._makeScreenshotV2(this.page, `${filePath}_main`, null);
await this._makeScreenshotV2(this.page, `${filePath}_main`, 2000);
const body = await this.page.content();
@ -278,10 +278,11 @@ class LVScrape extends Scraper {
const qualifyHoldings = await this.extractEntitySubSections(body, 'h2:contains("Qualifying holdings")');
// --
await jsonfile.writeFile(`${filePath}.json`, { details, marketSegments, relatedPersons, licenses, sanctions, qualifyHoldings });
await jsonfile.writeFile(`${filePath}.json`, { details, licenses, marketSegments, qualifyHoldings, relatedPersons, sanctions });
await this._randomWait(this.page, 3, 5);
if (details.hasOwnProperty('links')) {
await this.page._client.send('Page.setDownloadBehavior', { 'behavior': 'allow', 'downloadPath': this.path });
for(const items of details.links) {
@ -293,6 +294,7 @@ class LVScrape extends Scraper {
if (!err.message.includes('net::ERR_ABORTED') )
logger.error('grabLink', err);
});
await this._randomWait(this.page, 2, 3);
}
}

144
ncas/nl.js
View File

@ -3,7 +3,7 @@ const cheerio = require('cheerio');
const path = require('path');
const jsonfile = require('jsonfile');
const removeAccents = require('remove-accents-diacritics');
const logger = require('log4js').getLogger('NL');
const logger = require('log4js').getLogger('(NL)');
const url = require('url');
logger.level = process.env.LOGGER_LEVEL || 'warn';
@ -59,7 +59,7 @@ class NLScrape extends Scraper {
catch( err) {
logger.error(err);
}
return description;
}
@ -179,6 +179,7 @@ class NLScrape extends Scraper {
const noWhiteSpace = /\W/g;
const urlSections = ['WFTBI', 'WFTEG', 'WFTKF'];
const id = serviceObject.links[serviceObject.step].id;
const hash = serviceObject.links[serviceObject.step].hash;
logger.info(`Process V2 ${this.modeTitles[this.mode]} entity ${serviceObject.step + 1} of ${serviceObject.items} // ${id}`);
@ -188,7 +189,8 @@ class NLScrape extends Scraper {
const filename = this._makeFileName(entity);
const filePath = `${this.path}/${filename}`.substring(0, 240);
// DIN-329 - Trim a bit harder and append the unique hash
const filePath = `${`${this.path}/${filename}`.substring(0, 215) }_${hash}`;
await this.page.waitForSelector('#contentcolumn > div.interactive-tabs > ol > li:nth-child(2) > a', { 'visible':true, 'timeout':7500 }).then(async (elm) => {
await elm.click({ 'delay':Scraper.notARobot() });
@ -337,7 +339,7 @@ class NLScrape extends Scraper {
const options = await this.page.$$('#ddfilter option');
const selects = ['2:12(1) Carrying on the business of a bank', '2:13(1) Carrying on the business of a bank'];
const wantedOption = [];
wantedOption.push(selects[this.creditServices.step]);
wantedOption.push(selects[this.creditServices.menuStep]);
for (const item of options) {
const text = await this.page.evaluate(el => el.innerText, item);
const value = await this.page.evaluate(el => el.value, item);
@ -355,6 +357,18 @@ class NLScrape extends Scraper {
}
}
getQSIdShortHash(inStr) {
const extractor = /([^?=&]+)(=([^&]*))?/g;
const matched = inStr.match(extractor);
if (matched.length === 2)
return matched[1].slice(3, 11).toUpperCase();
else
return '';
}
/**
* Process WFTBI / Payment Services
* @returns {Promise<void>}
@ -366,10 +380,10 @@ class NLScrape extends Scraper {
const origUrl = await this.page.url();
const pageUrl = url.parse(origUrl);
if (pageUrl.query === null)
// we need to select the correct item from the dropdown.
if (pageUrl.query === null)
// we need to select the correct item from the dropdown.
this.initiateWFTBI();
else {
// crack query
@ -391,22 +405,26 @@ class NLScrape extends Scraper {
statutoryName = removeAccents.remove(statutoryName.trim()).replace(nonWhiteSpace, '_');
tradeName = removeAccents.remove(tradeName.trim()).replace(nonWhiteSpace, '_');
tradeName = removeAccents.remove(tradeName.trim()).replace(nonWhiteSpace, '_') || '';
const id = (statutoryName === tradeName) ? statutoryName : `${statutoryName}-${tradeName}`;
const statutoryNameExists = this.paymentServices.statutoryNames.indexOf(statutoryName.toLowerCase());
const id = (statutoryNameExists === -1) ? statutoryName : `${statutoryName}-${tradeName}`;
let href = cheerio(children.eq(0)).find('a').attr('href');
href = href.concat('&locale=en_GB');
// this is the one we want.
const hash = this.getQSIdShortHash(href);
this.paymentServices.links.push({ id, href });
href = href.concat('&locale=en_GB');
this.paymentServices.links.push({ id, href, hash });
if (statutoryNameExists === -1) this.paymentServices.statutoryNames.push(statutoryName.toLowerCase());
});
const next = $('a.next').attr('href') || '';
if (next !== '')
if (next !== '')
this._findAndClick('a.next');
else
else
this.emit('startProcessingPaymentServices');
}
}
@ -447,16 +465,19 @@ class NLScrape extends Scraper {
statutoryName = removeAccents.remove(statutoryName.trim()).replace(nonWhiteSpace, '_');
tradeName = removeAccents.remove(tradeName.trim()).replace(nonWhiteSpace, '_');
tradeName = removeAccents.remove(tradeName.trim()).replace(nonWhiteSpace, '_') || '';
// const id = `${statutoryName}-${tradeName}`;
const id = (statutoryName === tradeName) ? statutoryName : `${statutoryName}-${tradeName}`;
const statutoryNameExists = this.emoneyServices.statutoryNames.indexOf(statutoryName.toLowerCase());
const id = (statutoryNameExists === -1) ? statutoryName : `${statutoryName}-${tradeName}`;
let href = cheerio(children.eq(0)).find('a').attr('href');
href = href.concat('&locale=en_GB');
// this is the one we want.
const hash = this.getQSIdShortHash(href);
this.emoneyServices.links.push({ id, href });
href = href.concat('&locale=en_GB');
this.emoneyServices.links.push({ id, href, hash });
if (statutoryNameExists === -1) this.emoneyServices.statutoryNames.push(statutoryName.toLowerCase());
});
const next = $('a.next').attr('href') || '';
@ -506,19 +527,19 @@ class NLScrape extends Scraper {
statutoryName = removeAccents.remove(statutoryName.trim()).replace(nonWhiteSpace, '_');
tradeName = removeAccents.remove(tradeName.trim()).replace(nonWhiteSpace, '_');
tradeName = removeAccents.remove(tradeName.trim()).replace(nonWhiteSpace, '_') || '';
const id = (statutoryName === tradeName) ? statutoryName : `${statutoryName}-${tradeName}`;
const statutoryNameExists = this.creditServices.statutoryNames.indexOf(statutoryName.toLowerCase());
// const id = `${statutoryName}-${tradeName}`;
const id = (statutoryNameExists === -1) ? statutoryName : `${statutoryName}-${tradeName}`;
let href = cheerio(children.eq(0)).find('a').attr('href');
const hash = this.getQSIdShortHash(href);
href = href.concat('&locale=en_GB');
// this is the one we want.
logger.debug({ id, href });
this.creditServices.links.push({ id, href });
this.creditServices.links.push({ id, href, hash });
if (statutoryNameExists === -1) this.creditServices.statutoryNames.push(statutoryName.toLowerCase());
});
const next = $('a.next').attr('href') || '';
@ -526,8 +547,8 @@ class NLScrape extends Scraper {
if (next !== '')
this._findAndClick('a.next');
else
if (this.creditServices.step === 0) {
this.creditServices.step = 1;
if (this.creditServices.menuStep === 0) {
this.creditServices.menuStep = 1;
await this._goto(this.credit);
}
else
@ -591,24 +612,32 @@ class NLScrape extends Scraper {
* @returns {Promise<void>}
*/
async restart() {
// v2
logger.info(`Restarting ${this.modeTitles[this.mode]}`);
switch (this.mode) {
case 2:
this.emit('startProcessingCreditServices');
break;
case 1:
this.emit('startProcessingEMoneyServices');
break;
case 0:
default:
if (this.mode === 0)
if (this.paymentServices.links.length > 0) {
this.emit('startProcessingPaymentServices');
break;
}
}
else {
await this._goto(this.startPage);
}
else if(this.mode === 1)
if (this.emoneyServices.links.length > 0) {
this.emit('startProcessingEMoneyServices');
}
else {
await this._goto(this.emoneyUrl);
}
else if(this.mode === 2)
if (this.creditServices.links.length > 0) {
this.emit('startProcessingCreditServices');
}
else {
await this._goto(this.credit);
}
}
/**
@ -626,12 +655,13 @@ class NLScrape extends Scraper {
this.processNewPage();
});
const timeout = 90000;
const antiCollision = 125 + (Math.floor(Math.random() * (15 - 1)) * 500);
const timeout = 90000 + antiCollision;
setTimeout(async() => {
setTimeout(() => {
logger.warn('Attempting recovery..');
await this.restart();
this.restart();
}, timeout);
}
@ -726,29 +756,33 @@ class NLScrape extends Scraper {
this.mode = 0;
try {
this.paymentServices = {
'done' : false,
'items': 0,
'links': [],
'statutoryNames': [],
'step': 0,
'visited': false,
'done' : false
'visited': false
};
this.emoneyServices = {
'done' : false,
'items': 0,
'links': [],
'searchDone' : false,
'statutoryNames': [],
'step': 0,
'visited': false,
'done' : false,
'searchDone' : false
'visited': false
};
this.creditServices = {
'done' : false,
'items': 0,
'links': [],
'menuStep' : 0,
'searchDone' : false,
'statutoryNames': [],
'step': 0,
'visited': false,
'done' : false,
'searchDone' : false
'visited': false
};
this.startPage = 'https://www.dnb.nl/en/supervision/public-register/WFTBI/index.jsp';

353
ncas/no.js
View File

@ -4,7 +4,7 @@ const Scraper = require('../helpers/scraper');
const cheerio = require('cheerio');
const path = require('path');
const jsonfile = require('jsonfile');
const logger = require('log4js').getLogger('NO');
const logger = require('log4js').getLogger('(NO)');
const url = require('url');
const removeAccents = require('remove-accents-diacritics');
@ -14,7 +14,15 @@ class NOScrape extends Scraper {
constructor() {
super();
this.id = 'NO';
this.setID('NO');
// treat these elements as block boundaries when scraping crossborder permissions
this.blockBoundaries = 'div, li';
// override these values from the base class
this.modePrefix = ['ps_', 'ps__in_', 'em_', 'em__in_', 'ci_', 'ci__in_'];
this.modeNames = ['paymentServices', 'paymentServicesIncoming', 'emoneyServices', 'emoneyServicesIncoming', 'creditServices', 'creditServicesIncoming'];
this.modeTitles = ['Payment Service', 'Payment Service Incoming', 'EMoney', 'EMoney Incoming', 'Credit Services', 'Credit Services Incoming'];
this.on('done', () => {
this._done();
@ -31,6 +39,70 @@ class NOScrape extends Scraper {
});
}
_reduceWhiteSpace(text) {
return text.replace(/\s+/g, ' ').trim();
}
/**
*
* @param html
* @param selector
* @returns {Promise<void>}
*
* Finds elements in the `html` with the given `selector`, but returns only the uppermost matched elements,
* and not those that are nested within other matched elements.
*/
getUppermostElementsBySelector(html, selector) {
const $ = cheerio.load(html);
return $(selector).filter(function () {
return $(this).parents(selector).length === 0;
});
}
getTextNotInMatchingElements(html, selector) {
const $ = cheerio.load(html);
$(selector)
.remove()
.end();
return $.text();
}
recurseDOM(html, selector, level = 0) {
const currentLevel = level + 1;
const $ = cheerio.load(html);
const result = [];
const blocks = this.getUppermostElementsBySelector(html, selector);
for (let i = 0; i < blocks.length; i++) {
const block = blocks[i];
const rawName = this.getTextNotInMatchingElements($(block).html(), selector);
const name = this._reduceWhiteSpace(rawName);
const blockHtml = $(block).html();
const data = this.recurseDOM(blockHtml, selector, currentLevel);
if (data === null)
result.push(name);
else
result.push({
'name': name,
'data': data
});
}
if (result.length > 0)
return result;
return null;
}
/**
*
* @param html
@ -184,6 +256,23 @@ class NOScrape extends Scraper {
return newObj;
}
recurseCrossborderHtml(html) {
try {
// Wrap html in an outer div to give the recursion loop a root starting point
const wrappedHtml = `<div>${html}</div>`;
// recurse the dom, using the selectors we have defined as block boundaries
const results = this.recurseDOM(wrappedHtml, this.blockBoundaries);
// return the first item from our list (i.e. the outer div we added at the top of this function)
return results[0];
}
catch(err) {
logger.error('Error during recursion of cross-border HTML');
logger.error(err);
}
}
/**
*
* @param html
@ -191,40 +280,15 @@ class NOScrape extends Scraper {
*/
extractEntityDetailCrossBorder(html) {
try {
const newObj = { };
const $ = cheerio.load(html);
const header = $('h3.license-unit-label:contains("Cross-border services/classes")');
const detailBox = $(header).parent();
const children = $(detailBox).children();
const crossborder = this.recurseCrossborderHtml(detailBox.html());
let curLabel = '';
children.each(async (i, item) => {
const tagName = $(item).prop('tagName');
if (tagName === 'H3') {
curLabel = this._makeFieldName($(item).text());
if (!newObj.hasOwnProperty(curLabel))
newObj[curLabel] = [];
}
if (['SPAN', 'A', 'P'].indexOf(tagName) !== -1)
newObj[curLabel].push(this._cleanUp($(item).text()));
if(tagName === 'DIV' || tagName === 'UL') {
if (!newObj.hasOwnProperty('data'))
newObj['data'] = [];
const cbData = this.extractCrossBorderDetailsV2($(item).html());
newObj['data'].push(cbData);
}
});
return newObj;
return crossborder;
}
catch( err) {
logger.error(err);
@ -254,6 +318,17 @@ class NOScrape extends Scraper {
}
}
/**
*
* @param serviceObject
* @param elm
* @returns {Promise<void>}
*/
async selectCountryOption(serviceObject, elm) {
const countryOption = serviceObject.country;
await this.page.select('select.search-filter[aria-label="Choose country"]', countryOption);
}
/**
*
* @param html
@ -322,7 +397,7 @@ class NOScrape extends Scraper {
await this._randomWait(this.page, 5, 7);
this._makeScreenshotV2(this.page, `${this.path}/${filename}_main_${serviceObject.indexStep}`, null);
this._makeScreenshotV2(this.page, `${this.path}/${filename}_main_${serviceObject.indexStep}`, 2500);
this.emit('indexdone');
}
@ -346,6 +421,18 @@ class NOScrape extends Scraper {
logger.warn('Waiting for data timeout');
});
await this._randomWait(this.page, 3, 5);
await this.page.waitForSelector('select.search-filter[aria-label="Choose country"]', { 'visible':true, 'timeout':7500 }).then(async (elm) => {
await this.selectCountryOption(serviceObject, elm);
}).catch((e) => {
logger.error(e);
logger.warn('No country select');
});
// Give the country selection some time to filter the results
await this._randomWait(this.page, 3, 5);
await this.page.waitForSelector('#js-konsregList > div > div', { 'visible':true, 'timeout':7500 }).then(async (elm) => {
await this.processIndex(serviceObject);
}).catch((e) => {
@ -375,7 +462,8 @@ class NOScrape extends Scraper {
{ 'find':'h2:contains("Agency debt collection on behalf of others")', 'blockType':'Debt collection' },
{ 'find':'h2:contains("E-money institution")', 'blockType':'E-money institution' },
{ 'find':'h2:contains("Investment firm")', 'blockType':'h2:contains("Investment firm")' },
{ 'find':'h2:contains("Intermediator of loans and guarantees")', 'blockType':'Intermediator of loans and guarantees' }
{ 'find':'h2:contains("Intermediator of loans and guarantees")', 'blockType':'Intermediator of loans and guarantees' },
{ 'find':'h2:contains("Payment service provider with a limited authorisat")', 'blockType':'Payment service provider with a limited authorisat' }
];
@ -421,7 +509,7 @@ class NOScrape extends Scraper {
await this._randomWait(this.page, 5, 7);
await this._makeScreenshotV2(this.page, `${filePath}_main`, null);
await this._makeScreenshotV2(this.page, `${filePath}_main`, 2500);
const body = await this.page.content();
@ -460,13 +548,25 @@ class NOScrape extends Scraper {
break;
case 1:
await this.buildIndex(this.emoneyServices);
await this.buildIndex(this.paymentServicesIncoming);
break;
case 2:
await this.buildIndex(this.emoneyServices);
break;
case 3:
await this.buildIndex(this.emoneyServicesIncoming);
break;
case 4:
await this.buildIndex(this.creditServices);
break;
case 5:
await this.buildIndex(this.creditServicesIncoming);
break;
}
}
@ -482,13 +582,25 @@ class NOScrape extends Scraper {
break;
case 1:
await this.processEntityDetails(this.emoneyServices);
await this.processEntityDetails(this.paymentServicesIncoming);
break;
case 2:
await this.processEntityDetails(this.emoneyServices);
break;
case 3:
await this.processEntityDetails(this.emoneyServicesIncoming);
break;
case 4:
await this.processEntityDetails(this.creditServices);
break;
case 5:
await this.processEntityDetails(this.creditServicesIncoming);
break;
}
}
@ -553,13 +665,25 @@ class NOScrape extends Scraper {
break;
case 1:
this.emit('emoneyServicesDone');
this.emit('paymentServicesIncomingDone');
break;
case 2:
this.emit('emoneyServicesDone');
break;
case 3:
this.emit('emoneyServicesIncomingDone');
break;
case 4:
this.emit('creditServicesDone');
break;
case 5:
this.emit('creditServicesIncomingDone');
break;
}
});
@ -578,6 +702,21 @@ class NOScrape extends Scraper {
await this._goto(newUrl);
});
this.on('psincindexdone', async () => {
let newUrl;
this.paymentServicesIncoming.items = this.paymentServicesIncoming.links.length;
logger.info(`${this.paymentServicesIncoming.items} items indexed`);
this.paymentServicesIncoming.indexStep++;
if (this.paymentServicesIncoming.indexStep >= this.paymentServicesIncoming.wanted.length)
newUrl = this.paymentServicesIncoming.links[this.paymentServicesIncoming.step].href;
else
newUrl = this.paymentServicesIncoming.urls[0];
await this._goto(newUrl);
});
this.on('emindexdone', async () => {
let newUrl;
this.emoneyServices.items = this.emoneyServices.links.length;
@ -592,6 +731,20 @@ class NOScrape extends Scraper {
await this._goto(newUrl);
});
this.on('emincindexdone', async () => {
let newUrl;
this.emoneyServicesIncoming.items = this.emoneyServicesIncoming.links.length;
logger.info(`${this.emoneyServicesIncoming.items} items indexed`);
this.emoneyServicesIncoming.indexStep++;
if (this.emoneyServicesIncoming.indexStep >= this.emoneyServicesIncoming.urls.length)
newUrl = this.emoneyServicesIncoming.links[this.emoneyServicesIncoming.step].href;
else
newUrl = this.emoneyServicesIncoming.urls[this.emoneyServicesIncoming.indexStep];
await this._goto(newUrl);
});
this.on('ciindexdone', async () => {
let newUrl;
this.creditServices.items = this.creditServices.links.length;
@ -605,6 +758,20 @@ class NOScrape extends Scraper {
await this._goto(newUrl);
});
this.on('ciincindexdone', async () => {
let newUrl;
this.creditServicesIncoming.items = this.creditServicesIncoming.links.length;
logger.info(`${this.creditServicesIncoming.items} items indexed`);
this.creditServicesIncoming.indexStep++;
if (this.creditServicesIncoming.indexStep >= this.creditServicesIncoming.urls.length)
newUrl = this.creditServicesIncoming.links[this.creditServicesIncoming.step].href;
else
newUrl = this.creditServicesIncoming.urls[this.creditServicesIncoming.indexStep];
await this._goto(newUrl);
});
this.on('indexdone', async () => {
switch (this.mode) {
@ -614,13 +781,25 @@ class NOScrape extends Scraper {
break;
case 1:
this.emit('emindexdone');
this.emit('psincindexdone');
break;
case 2:
this.emit('emindexdone');
break;
case 3:
this.emit('emincindexdone');
break;
case 4:
this.emit('ciindexdone');
break;
case 5:
this.emit('ciincindexdone');
break;
}
});
@ -634,7 +813,24 @@ class NOScrape extends Scraper {
this.mode++;
this.inProgress = false;
await this._goto(this.emoneyServices.urls[0]);
await this._goto(this.paymentServicesIncoming.urls[0]);
}
catch (e) {
logger.error(e);
}
});
this.on('paymentServicesIncomingDone', async () => {
logger.warn('paymentServicesIncomingDone');
try{
this.paymentServicesIncoming.done = true;
jsonfile.writeFileSync(`${this.path}/paymentServicesIncoming.json`, { 'links': this.paymentServicesIncoming.links });
jsonfile.writeFileSync(`${this.debugPath}/paymentServicesIncoming.json`, this.paymentServicesIncoming);
this.mode++;
this.inProgress = false;
await this._goto(this.emoneyServicesIncoming.urls[0]);
}
catch (e) {
logger.error(e);
@ -650,13 +846,29 @@ class NOScrape extends Scraper {
this.mode++;
this.inProgress = false;
await this._goto(this.creditServices.urls[0]);
await this._goto(this.emoneyServicesIncoming.urls[0]);
}
catch (e) {
logger.error(e);
}
});
this.on('emoneyServicesIncomingDone', async () => {
logger.warn('emoneyServicesIncomingDone');
try{
this.emoneyServicesIncoming.done = true;
jsonfile.writeFileSync(`${this.path}/emoneyServicesIncoming.json`, { 'links':this.emoneyServicesIncoming.links });
jsonfile.writeFileSync(`${this.debugPath}/emoneyServicesIncoming.json`, this.emoneyServicesIncoming);
this.mode++;
this.inProgress = false;
await this._goto(this.creditServices.urls[0]);
}
catch (e) {
logger.error(e);
}
});
this.on('creditServicesDone', async () => {
logger.warn('creditServicesDone');
try{
@ -666,6 +878,22 @@ class NOScrape extends Scraper {
this.mode++;
this.inProgress = false;
await this._goto(this.creditServicesIncoming.urls[0]);
}
catch (e) {
logger.error(e);
}
});
this.on('creditServicesIncomingDone', async () => {
logger.warn('creditServicesIncomingDone');
try{
this.creditServicesIncoming.done = true;
jsonfile.writeFileSync(`${this.path}/creditServicesIncoming.json`, { 'links':this.creditServicesIncoming.links });
jsonfile.writeFileSync(`${this.debugPath}/creditServicesIncoming.json`, this.creditServicesIncoming);
this.mode++;
this.inProgress = false;
this.emit('done');
}
catch (e) {
@ -694,6 +922,21 @@ class NOScrape extends Scraper {
'done' : false,
'urls': ['https://www.finanstilsynet.no/en/finanstilsynets-registry/'],
'wanted' : ['Payment institution', 'Agent of payment institution (company)', 'Payment service provider with a limited authorisat'],
'country' : 'NOR',
'sections' : [],
'sectionLinks' : []
};
this.paymentServicesIncoming = {
'items': 0,
'links': [],
'step': 0,
'indexStep': 0,
'visited': false,
'done' : false,
'urls': ['https://www.finanstilsynet.no/en/finanstilsynets-registry/'],
'wanted' : ['Payment institution', 'Agent of payment institution (company)', 'Payment service provider with a limited authorisat'],
'country' : 'NOTNOR',
'sections' : [],
'sectionLinks' : []
};
@ -707,6 +950,21 @@ class NOScrape extends Scraper {
'done' : false,
'urls': ['https://www.finanstilsynet.no/en/finanstilsynets-registry/'],
'wanted' : ['E-money institution'],
'country' : 'NOR',
'sections' : [],
'sectionLinks' : []
};
this.emoneyServicesIncoming = {
'items': 0,
'links': [],
'step': 0,
'indexStep': 0,
'visited': false,
'done' : false,
'urls': ['https://www.finanstilsynet.no/en/finanstilsynets-registry/'],
'wanted' : ['E-money institution'],
'country' : 'NOTNOR',
'sections' : [],
'sectionLinks' : []
};
@ -722,13 +980,28 @@ class NOScrape extends Scraper {
'started': false,
'urls': ['https://www.finanstilsynet.no/en/finanstilsynets-registry/'],
'wanted' : ['Bank', 'Branch of foreign credit institution', 'Credit Institution', 'Savings bank foundation'],
'country' : 'NOR',
'sections' : [],
'sectionLinks' : []
};
this.creditServicesIncoming = {
'items': 0,
'links': [],
'step': 0,
'indexStep': 0,
'visited': false,
'done' : false,
'searchDone' : false,
'started': false,
'urls': ['https://www.finanstilsynet.no/en/finanstilsynets-registry/'],
'wanted' : ['Bank', 'Branch of foreign credit institution', 'Credit Institution', 'Savings bank foundation'],
'country' : 'NOTNOR',
'sections' : [],
'sectionLinks' : []
};
this.startPage = this.paymentServices.urls[0];
this.emoneyUrl = this.emoneyServices.urls[0];
this.credit = this.creditServices.urls[0];
this.setPath(path.resolve(`${__dirname }/../artefacts/NO/FS`));

28
ncas/pl.js
View File

@ -704,7 +704,7 @@ class PLScrape extends Scraper {
}
catch( err) {
logger.error(err);
this.emit('recover');
this.emit('backoff');
}
}
@ -926,7 +926,10 @@ class PLScrape extends Scraper {
});
if (doIndex)
await this.processEntityIndex(this.paymentServices);
await this.processEntityIndex(this.paymentServices).catch(async (err) => {
logger.error('processEntityIndex catch: ', err);
this.emit('restart');
});
}
/**
@ -1021,7 +1024,7 @@ class PLScrape extends Scraper {
if (pageUrl.href === 'chrome-error://chromewebdata/') {
logger.warn('Directed to: chrome-error://chromewebdata/');
this.emit('recover');
this.emit('backoff');
return;
}
@ -1102,7 +1105,7 @@ class PLScrape extends Scraper {
*
* @returns {Promise<void>}
*/
async restart() {
async _restart() {
logger.warn(`Tryng to restart ${this.modeTitles[this.mode]}`);
if (this.mode === 0) {
@ -1145,9 +1148,10 @@ class PLScrape extends Scraper {
if (this.backOffStep > this.backOffLimit) this.backOffStep = this.backOffLimit;
logger.warn(`Backing off for ${this.backOffStep * 5} minutes..`);
const timeout = (60 * 1000) * (this.backOffStep * 5);
const timeout = 300000; // (this.backOffStep * 5) * 60000;
await this._uploadError();
logger.warn('timeout', timeout);
// await this._uploadError();
this.backOffTimer = setTimeout(() => {
this.emit('restart');
@ -1166,15 +1170,19 @@ class PLScrape extends Scraper {
}, 2500));
this.on('stall', () => {
this.backoff();
this.emit('backoff');
});
this.on('backoff', () => {
this.on('backoff', this._debounce( () => {
this.backoff();
});
}, 10000));
/* this.on('backoff', () => {
this.backoff();
});*/
this.on('restart', async () => {
await this.restart();
await this._restart();
});
this.on('entityComplete', () => {

6
ncas/pt.js
View File

@ -264,13 +264,15 @@ class PTScrape extends Scraper {
const pageUrl = url.parse(await this.page.url());
if (pageUrl.href === 'chrome-error://chromewebdata/') {
logger.warn('Directed to: chrome-error://chromewebdata/');
if (pageUrl.href === 'chrome-error://chromewebdata/' || pageUrl.pathname === null) {
logger.warn('Directed to: chrome-error://chromewebdata/ or null');
this.emit('recover');
return;
}
logger.debug('processNewPage', pageUrl.pathname);
const splitPath = pageUrl.pathname.match(pathSplitter);
const pathname = splitPath[0];

230
ncas/sk.js
View File

@ -168,62 +168,68 @@ class SKScrape extends Scraper {
* @returns {Promise<void>}
*/
async processEntityIndex(serviceObject) {
const fields = ['referenceNumber', 'businessName', 'address', 'start', 'end', 'reason'];
try{
const fields = ['referenceNumber', 'businessName', 'address', 'start', 'end', 'reason'];
const mouseDownDuration = Scraper.notARobot();
if (serviceObject.visited === false) {
logger.debug('Preparing...');
const mouseDownDuration = Scraper.notARobot();
if (serviceObject.visited === false) {
logger.debug('Preparing...');
await this.page.waitForSelector('table#Subjects', { 'visible':true }).then(async () => {
await this.entityIndexFirstPass(serviceObject);
}).catch(() => {
logger.error('Table failed to render');
});
}
await this.page.waitForSelector('table#Subjects', { 'visible':true }).then(async () => {
await this.entityIndexFirstPass(serviceObject);
}).catch(() => {
logger.error('Table failed to render');
});
}
if (serviceObject.visited === true) {
serviceObject.currentMetaIndex = serviceObject.step % 10;
if (serviceObject.visited === true) {
serviceObject.currentMetaIndex = serviceObject.step % 10;
if ((serviceObject.step ) >= serviceObject.currentPageMax) {
const nextButton = await this.page.$$('#Subjects_next');
if ((serviceObject.step ) >= serviceObject.currentPageMax) {
const nextButton = await this.page.$$('#Subjects_next');
const buttonClasses = await this.page.$eval('#Subjects_next', e => e.getAttribute('class'));
const buttonClasses = await this.page.$eval('#Subjects_next', e => e.getAttribute('class'));
if (buttonClasses.split(' ').indexOf('disabled') === -1) {
// we need a click..
nextButton[0].click({ 'delay':mouseDownDuration });
if (buttonClasses.split(' ').indexOf('disabled') === -1) {
// we need a click..
nextButton[0].click({ 'delay':mouseDownDuration });
await this._randomWait(this.page, 3, 5);
serviceObject.visited = false;
this.emit('entityIndex');
}
else {
logger.debug('I think we are done here...');
this.emit('serviceDone');
}
}
else {
await this.page.waitForSelector('#Subjects > tbody');
const wantedRow = await this.page.$$(`#Subjects > tbody > tr:nth-child(${serviceObject.currentMetaIndex + 1})`);
const htmlRow = await this.page.evaluate(el => el.outerHTML, wantedRow[0]);
const $ = cheerio.load(`<table>${htmlRow}</table>`);
const cells = $('td');
serviceObject.current = {};
cells.each((index, item) => {
serviceObject.current[ fields[index] ] = $(item).text();
});
await this._randomWait(this.page, 3, 5);
serviceObject.visited = false;
this.emit('entityIndex');
}
else {
logger.debug('I think we are done here...');
this.emit('serviceDone');
await wantedRow[0].click({ 'delay':mouseDownDuration });
}
}
else {
await this.page.waitForSelector('#Subjects > tbody');
const wantedRow = await this.page.$$(`#Subjects > tbody > tr:nth-child(${serviceObject.currentMetaIndex + 1})`);
const htmlRow = await this.page.evaluate(el => el.outerHTML, wantedRow[0]);
const $ = cheerio.load(`<table>${htmlRow}</table>`);
const cells = $('td');
serviceObject.current = {};
cells.each((index, item) => {
serviceObject.current[ fields[index] ] = $(item).text();
});
await this._randomWait(this.page, 3, 5);
await wantedRow[0].click({ 'delay':mouseDownDuration });
}
}
catch( err) {
logger.error(err);
this.emit('recover');
}
}
@ -386,76 +392,82 @@ class SKScrape extends Scraper {
* @returns {Promise<void>}
*/
async processEntityDetail(serviceObject) {
// level0 sublicctrl sublicctrl1 odd
// level0 sublicctrl sublicctrl1 odd sublicshow shown
try{
// level0 sublicctrl sublicctrl1 odd
// level0 sublicctrl sublicctrl1 odd sublicshow shown
// expand all accordians
// expand all accordians
const rows = await this.page.$$('tr.sublicctrl');
const rows = await this.page.$$('tr.sublicctrl');
for (const item of rows) {
const cls = await this.page.evaluate(el => el.getAttribute('class'), item);
if (!cls.includes('shown'))
for (const item of rows) {
const cls = await this.page.evaluate(el => el.getAttribute('class'), item);
if (!cls.includes('shown'))
await item.click({ 'delay':Scraper.notARobot() });
}
await this.page.waitForSelector('#Licenses > tbody > tr.level1.shown.sublichide1.sllhidectrl.sllhidectrl1', { 'timeout':7500 }).then(async (elm) => {
await elm.click({ 'delay':Scraper.notARobot() });
}).catch(() => {
logger.debug('No License information');
});
await this._microWait(this.page, 5);
// expand all viewable anchors
const wantedAnchors = await this.page.$$('.row a');
for (const item of wantedAnchors) {
const exItem = this._cleanUp(await this.page.evaluate(el => el.text, item));
if (exItem === 'View') {
await item.hover().catch((e) => {
logger.warn('Hover failed', e.name);
});
await item.click({ 'delay': Scraper.notARobot() }).catch((e) => {
logger.debug('View click failed', e.name);
});
await item.click({ 'delay':Scraper.notARobot() });
}
await this.page.waitForSelector('#Licenses > tbody > tr.level1.shown.sublichide1.sllhidectrl.sllhidectrl1', { 'timeout':7500 }).then(async (elm) => {
await elm.click({ 'delay':Scraper.notARobot() });
}).catch(() => {
logger.debug('No License information');
});
await this._microWait(this.page, 5);
// expand all viewable anchors
const wantedAnchors = await this.page.$$('.row a');
for (const item of wantedAnchors) {
const exItem = this._cleanUp(await this.page.evaluate(el => el.text, item));
if (exItem === 'View') {
await item.hover().catch((e) => {
logger.warn('Hover failed', e.name);
});
await item.click({ 'delay': Scraper.notARobot() }).catch((e) => {
logger.debug('View click failed', e.name);
});
}
}
const entityName = `${serviceObject.current.businessName}_${serviceObject.current.referenceNumber}`;
const fileName = this._makeFileName(entityName);
const filePath = await this._makeFilePath(entityName);
serviceObject.current.fileName = fileName;
await this._randomWait(this.page, 2, 2);
await this.page.focus('h3.page-header');
await this._makeScreenshotV2(this.page, `${filePath}_main`, null);
await this.page.waitForSelector('body > div.container > form.form-horizontal > table', { 'timeout':7500 }).then(async (elm) => {
logger.debug('prep for processEntityDetailBasicDetails');
const htmlBlock = await this.page.evaluate(el => el.outerHTML, elm);
const $ = cheerio.load(htmlBlock);
serviceObject.current.basicDetails = await this.processEntityDetailBasicDetails($);
});
await this.page.waitForSelector('#Licenses').then(async (elm) => {
logger.debug('prep for processEntityDetailTableV2');
const htmlBlock = await this.page.evaluate(el => el.outerHTML, elm);
const $ = cheerio.load(htmlBlock);
serviceObject.current.entityDetails = await this.processEntityDetailTableV2($);
});
this.entityCompleter(serviceObject);
}
catch( err) {
logger.error(err);
this.emit('recover');
}
const entityName = `${serviceObject.current.businessName}_${serviceObject.current.referenceNumber}`;
const fileName = this._makeFileName(entityName);
const filePath = await this._makeFilePath(entityName);
serviceObject.current.fileName = fileName;
await this._randomWait(this.page, 2, 2);
await this.page.focus('h3.page-header');
await this._makeScreenshotV2(this.page, `${filePath}_main`, null);
await this.page.waitForSelector('body > div.container > form.form-horizontal > table', { 'timeout':7500 }).then(async (elm) => {
logger.debug('prep for processEntityDetailBasicDetails');
const htmlBlock = await this.page.evaluate(el => el.outerHTML, elm);
const $ = cheerio.load(htmlBlock);
serviceObject.current.basicDetails = await this.processEntityDetailBasicDetails($);
});
await this.page.waitForSelector('#Licenses').then(async (elm) => {
logger.debug('prep for processEntityDetailTableV2');
const htmlBlock = await this.page.evaluate(el => el.outerHTML, elm);
const $ = cheerio.load(htmlBlock);
serviceObject.current.entityDetails = await this.processEntityDetailTableV2($);
});
this.entityCompleter(serviceObject);
}
/**

147
package-lock.json generated
View File

@ -353,7 +353,7 @@
},
"ansi-colors": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-1.1.0.tgz",
"resolved": "http://registry.npmjs.org/ansi-colors/-/ansi-colors-1.1.0.tgz",
"integrity": "sha512-SFKX67auSNoVR38N3L+nvsPjOE0bybKTYbkf5tRvushrAPQ9V75huw0ZxBkKVeRU9kqH3d6HA4xTckbwZ4ixmA==",
"dev": true,
"requires": {
@ -1253,7 +1253,7 @@
"dependencies": {
"uuid": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz",
"resolved": "http://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz",
"integrity": "sha1-Z+LoY3lyFVMN/zGOW/nc6/1Hsho="
}
}
@ -1690,7 +1690,7 @@
},
"duplexer": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz",
"resolved": "http://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz",
"integrity": "sha1-rOb/gIwc5mtX0ev5eXessCM0z8E=",
"dev": true
},
@ -1711,7 +1711,7 @@
},
"readable-stream": {
"version": "1.1.14",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
"resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
"integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=",
"dev": true,
"requires": {
@ -2816,12 +2816,6 @@
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz",
"integrity": "sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w=="
},
"get-own-enumerable-property-symbols": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.0.tgz",
"integrity": "sha512-CIJYJC4GGF06TakLg8z4GQKvDsx9EMspVxOYih7LerEL/WosUnFIww45CGfxfeKHqlg3twgUrYRT1O3WQqjGCg==",
"dev": true
},
"get-ssl-certificate": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/get-ssl-certificate/-/get-ssl-certificate-2.3.1.tgz",
@ -2979,7 +2973,7 @@
},
"readable-stream": {
"version": "1.0.34",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
"resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
"integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=",
"dev": true,
"requires": {
@ -3103,7 +3097,7 @@
},
"lodash": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-1.0.2.tgz",
"resolved": "http://registry.npmjs.org/lodash/-/lodash-1.0.2.tgz",
"integrity": "sha1-j1dWDIO1n8JwvT1WG2kAQ0MOJVE=",
"dev": true
},
@ -3156,7 +3150,7 @@
},
"got": {
"version": "6.7.1",
"resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz",
"resolved": "http://registry.npmjs.org/got/-/got-6.7.1.tgz",
"integrity": "sha1-JAzQV4WpoY5WHcG0S0HHY+8ejbA=",
"requires": {
"create-error-class": "^3.0.0",
@ -3179,7 +3173,7 @@
},
"gulp": {
"version": "3.9.1",
"resolved": "https://registry.npmjs.org/gulp/-/gulp-3.9.1.tgz",
"resolved": "http://registry.npmjs.org/gulp/-/gulp-3.9.1.tgz",
"integrity": "sha1-VxzkWSjdQK9lFPxAEYZgFsE4RbQ=",
"dev": true,
"requires": {
@ -3212,7 +3206,7 @@
},
"chalk": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
"resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
"integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
"dev": true,
"requires": {
@ -3225,19 +3219,19 @@
},
"minimist": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
"resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
"integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=",
"dev": true
},
"semver": {
"version": "4.3.6",
"resolved": "https://registry.npmjs.org/semver/-/semver-4.3.6.tgz",
"resolved": "http://registry.npmjs.org/semver/-/semver-4.3.6.tgz",
"integrity": "sha1-MAvG4OhjdPe6YQaLWx7NV/xlMto=",
"dev": true
},
"strip-ansi": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
"resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
"integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
"dev": true,
"requires": {
@ -3282,13 +3276,13 @@
},
"async": {
"version": "0.9.2",
"resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz",
"resolved": "http://registry.npmjs.org/async/-/async-0.9.2.tgz",
"integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=",
"dev": true
},
"bl": {
"version": "0.9.5",
"resolved": "https://registry.npmjs.org/bl/-/bl-0.9.5.tgz",
"resolved": "http://registry.npmjs.org/bl/-/bl-0.9.5.tgz",
"integrity": "sha1-wGt5evCF6gC8Unr8jvzxHeIjIFQ=",
"dev": true,
"requires": {
@ -3346,7 +3340,7 @@
},
"lodash": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-3.2.0.tgz",
"resolved": "http://registry.npmjs.org/lodash/-/lodash-3.2.0.tgz",
"integrity": "sha1-S/UKMkP5rrC6xBpV09WZBnWkYvs=",
"dev": true
},
@ -3361,7 +3355,7 @@
},
"readable-stream": {
"version": "1.0.34",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
"resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
"integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=",
"dev": true,
"requires": {
@ -3379,7 +3373,7 @@
},
"tar-stream": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.1.5.tgz",
"resolved": "http://registry.npmjs.org/tar-stream/-/tar-stream-1.1.5.tgz",
"integrity": "sha1-vpIYwTDCACnhB7D5Z/sj3gV50Tw=",
"dev": true,
"requires": {
@ -3461,29 +3455,6 @@
"through2": "^2.0.1"
}
},
"gulp-changed-in-place": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/gulp-changed-in-place/-/gulp-changed-in-place-2.3.0.tgz",
"integrity": "sha1-3kFQwnbYwAUkUcry8MfSvhfnaIg=",
"dev": true,
"requires": {
"through2": "^2.0.0"
}
},
"gulp-debug": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/gulp-debug/-/gulp-debug-4.0.0.tgz",
"integrity": "sha512-cn/GhMD2nVZCVxAl5vWao4/dcoZ8wUJ8w3oqTvQaGDmC1vT7swNOEbhQTWJp+/otKePT64aENcqAQXDcdj5H1g==",
"dev": true,
"requires": {
"chalk": "^2.3.0",
"fancy-log": "^1.3.2",
"plur": "^3.0.0",
"stringify-object": "^3.0.0",
"through2": "^2.0.0",
"tildify": "^1.1.2"
}
},
"gulp-gzip": {
"version": "1.4.2",
"resolved": "https://registry.npmjs.org/gulp-gzip/-/gulp-gzip-1.4.2.tgz",
@ -3651,7 +3622,7 @@
},
"chalk": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
"resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
"integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
"dev": true,
"requires": {
@ -3670,7 +3641,7 @@
},
"minimist": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
"resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
"integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=",
"dev": true
},
@ -3682,7 +3653,7 @@
},
"strip-ansi": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
"resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
"integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
"dev": true,
"requires": {
@ -3974,12 +3945,6 @@
"resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz",
"integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo="
},
"irregular-plurals": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/irregular-plurals/-/irregular-plurals-2.0.0.tgz",
"integrity": "sha512-Y75zBYLkh0lJ9qxeHlMjQ7bSbyiSqNW/UOPWDmzC7cXskL1hekSITh1Oc6JV0XCWWZ9DE8VYSB71xocLk3gmGw==",
"dev": true
},
"is-absolute": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-absolute/-/is-absolute-1.0.0.tgz",
@ -4140,7 +4105,7 @@
},
"is-obj": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz",
"resolved": "http://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz",
"integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8="
},
"is-path-cwd": {
@ -4192,12 +4157,6 @@
"has": "^1.0.1"
}
},
"is-regexp": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-regexp/-/is-regexp-1.0.0.tgz",
"integrity": "sha1-/S2INUXEa6xaYz57mgnof6LLUGk=",
"dev": true
},
"is-relative": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-relative/-/is-relative-1.0.0.tgz",
@ -4767,11 +4726,6 @@
"p-is-promise": "^2.0.0"
}
},
"memory": {
"version": "0.0.3",
"resolved": "https://registry.npmjs.org/memory/-/memory-0.0.3.tgz",
"integrity": "sha1-zgCclqXIreLyz0psmmlCprX+JvU="
},
"meow": {
"version": "3.7.0",
"resolved": "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz",
@ -5002,11 +4956,6 @@
"lower-case": "^1.1.1"
}
},
"node-free": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/node-free/-/node-free-1.0.0.tgz",
"integrity": "sha1-0rgIX0fCNh7dGfX/OpdKnLpJiwc="
},
"node-int64": {
"version": "0.3.3",
"resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.3.3.tgz",
@ -6567,7 +6516,7 @@
},
"pause-stream": {
"version": "0.0.11",
"resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz",
"resolved": "http://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz",
"integrity": "sha1-/lo0sMvOErWqaitAPuLnO2AvFEU=",
"dev": true,
"requires": {
@ -6684,15 +6633,6 @@
}
}
},
"plur": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/plur/-/plur-3.1.1.tgz",
"integrity": "sha512-t1Ax8KUvV3FFII8ltczPn2tJdjqbd1sIzu6t4JL7nQ3EyeL/lTrj5PWKb06ic5/6XYDr65rQ4uzQEGN70/6X5w==",
"dev": true,
"requires": {
"irregular-plurals": "^2.0.0"
}
},
"pm2": {
"version": "3.5.0",
"resolved": "https://registry.npmjs.org/pm2/-/pm2-3.5.0.tgz",
@ -6834,7 +6774,7 @@
},
"pretty-hrtime": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz",
"resolved": "http://registry.npmjs.org/pretty-hrtime/-/pretty-hrtime-1.0.3.tgz",
"integrity": "sha1-t+PqQkNaTJsnWdmeDyAesZWALuE=",
"dev": true
},
@ -6929,20 +6869,20 @@
"resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.14.0.tgz",
"integrity": "sha512-SayS2wUX/8LF8Yo2Rkpc5nkAu4Jg3qu+OLTDSOZtisVQMB2Z5vjlY2TdPi/5CgZKiZroYIiyUN3sRX63El9iaw==",
"requires": {
"debug": "^4.1.0",
"extract-zip": "^1.6.6",
"https-proxy-agent": "^2.2.1",
"debug": "^3.1.0",
"extract-zip": "^1.6.5",
"https-proxy-agent": "^2.1.0",
"mime": "^2.0.3",
"progress": "^2.0.1",
"progress": "^2.0.0",
"proxy-from-env": "^1.0.0",
"rimraf": "^2.6.1",
"ws": "^6.1.0"
"ws": "^3.0.0"
},
"dependencies": {
"debug": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
"version": "3.2.6",
"resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
"integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
"requires": {
"ms": "^2.1.1"
}
@ -6957,7 +6897,9 @@
"resolved": "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz",
"integrity": "sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA==",
"requires": {
"async-limiter": "~1.0.0"
"async-limiter": "~1.0.0",
"safe-buffer": "~5.1.0",
"ultron": "~1.1.0"
}
}
}
@ -7669,7 +7611,7 @@
"dependencies": {
"ansi-regex": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-0.2.1.tgz",
"resolved": "http://registry.npmjs.org/ansi-regex/-/ansi-regex-0.2.1.tgz",
"integrity": "sha1-DY6UaWej2BQ/k+JOKYUl/BsiNfk=",
"dev": true
},
@ -7681,7 +7623,7 @@
},
"chalk": {
"version": "0.5.1",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-0.5.1.tgz",
"resolved": "http://registry.npmjs.org/chalk/-/chalk-0.5.1.tgz",
"integrity": "sha1-Zjs6ZItotV0EaQ1JFnqoN4WPIXQ=",
"dev": true,
"requires": {
@ -7709,7 +7651,7 @@
},
"strip-ansi": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-0.3.0.tgz",
"resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-0.3.0.tgz",
"integrity": "sha1-JfSOoiynkYfzF0pNuHWTR7sSYiA=",
"dev": true,
"requires": {
@ -7731,7 +7673,7 @@
},
"stream-combiner": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz",
"resolved": "http://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz",
"integrity": "sha1-rsjLrBd7Vrb0+kec7YwZEs7lKFg=",
"dev": true,
"requires": {
@ -7813,17 +7755,6 @@
"safe-buffer": "~5.1.0"
}
},
"stringify-object": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/stringify-object/-/stringify-object-3.3.0.tgz",
"integrity": "sha512-rHqiFh1elqCQ9WPLIC8I0Q/g/wj5J1eMkyoiD6eoQApWHP0FtlK7rqnhmabL5VUY9JQCcqwwvlOaSuutekgyrw==",
"dev": true,
"requires": {
"get-own-enumerable-property-symbols": "^3.0.0",
"is-obj": "^1.0.1",
"is-regexp": "^1.0.0"
}
},
"strip-ansi": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
@ -8579,7 +8510,7 @@
},
"readable-stream": {
"version": "1.0.34",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
"resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.0.34.tgz",
"integrity": "sha1-Elgg40vIQtLyqq+v5MKRbuMsFXw=",
"dev": true,
"requires": {

6
package.json
View File

@ -1,7 +1,7 @@
{
"name": "obdfcascrape",
"version": "1.0.0",
"description": "",
"description": "Scraping system to extract data from European Bank and Financial Services",
"main": "index.js",
"scripts": {
"test": "nyc tape tests/**/*.js",
@ -15,7 +15,7 @@
"malta": "node mt.js",
"debuglogs": "node debuglogs.js"
},
"author": "",
"author": "Martin Donnelly",
"license": "ISC",
"dependencies": {
"archiver": "^2.1.1",
@ -39,7 +39,7 @@
"moment": "^2.24.0",
"node-free": "^1.0.0",
"pm2": "^3.5.0",
"puppeteer": "^1.14.0",
"puppeteer": "1.15.0",
"remove-accents-diacritics": "^1.0.2",
"request": "^2.88.0",
"tld-extract": "^1.0.1",

22
tests/data/at/ent_001.html Normal file
View File

@ -0,0 +1,22 @@
<h3>
<span class="sr-only">Intermarket Bank AG Wien</span>
<a href="#" class="company-details-toggle" rel="nofollow" title="Show Details" role="presentation" aria-hidden="true">
<span class="glyphicon indicator glyphicon-chevron-down" aria-hidden="true"></span>
Intermarket Bank AG </a>
</h3>
<div class="company-details">
<ul>
<li><strong>Category:</strong> Banks - Banks licensed in Austria<br>Banks - Austrian Banks in the EEA (freedom to provide services)<br></li><li><strong>Address:</strong> Am Belvedere 1&nbsp;|&nbsp;1100&nbsp;Wien | Austria</li><li><strong>Contact:</strong><ul><li><strong>Phone:</strong> +43 5010028900</li><li><strong>Email:</strong> <a href="mailto:Meldewesen@Intermarket.at">Meldewesen@Intermarket.at</a></li><li><strong>Web:</strong> <a href="http://www.intermarket.at" target="_blank" rel="noopener">www.intermarket.at</a></li></ul></li><li><strong>Legal Identifiers:</strong><ul><li><strong>Bank identification number:</strong> 73600</li><li><strong>Commercial register number:</strong> 94144y</li></ul></li>
</ul>
<p class="toggle-document-description-wrap"><i class="fa fa-eye" aria-hidden="true"></i> <a href="#" data-toggle="modal" data-target="#document-description-5ca20b072a24b">Show licenses</a></p><div class="modal fade document-description" id="document-description-5ca20b072a24b" role="dialog" tabindex="-1" aria-labelledby="modal-label-5ca20b072a24b"><div class="modal-dialog modal-md" role="document"><div class="modal-content"><div class="modal-header"><h6 class="modal-title" id="modal-label-5ca20b072a24b"><span class="sr-only">Description for:</span> Licenses for&nbsp;Intermarket Bank AG - Wien</h6></div><div class="modal-body"><h4>§ 1 Abs. 1 Z 1 BWG</h4><p>Die Entgegennahme fremder Gelder zur Verwaltung oder als Einlage (Einlagengeschäft);
ausschließlich des Interbankenmarktes.</p><h4>§ 1 Abs. 1 Z 3 BWG</h4><p>Der Abschluss von Geldkreditverträgen und die Gewährung von Gelddarlehen (Kreditgeschäft)</p><h4>§ 1 Abs. 1 Z 4 BWG</h4><p>Der Kauf von Schecks und Wechseln, insbesondere die Diskontierung von Wechseln (Diskontgeschäft)</p><h4>§ 1 Abs. 1 Z 7 BWG</h4><p>Der Handel auf eigene oder fremde Rechnung mit<br> a) ausländischen Zahlungsmitteln (Devisen- und Valutengeschäft);</p><h4>§ 1 Abs. 1 Z 8 BWG</h4><p>Die Übernahme von Bürgschaften, Garantien und sonstigen Haftungen für andere, sofern die übernommene Verpflichtung auf Geldleistungen lautet (Garantiegeschäft)</p><h4>§ 1 Abs. 1 Z 16 BWG</h4><p>Der Ankauf von Forderungen aus Warenlieferungen oder Dienstleistungen, die Übernahme des Risikos der Einbringlichkeit solcher Forderungen ¿ ausgenommen die Kreditversicherung - und im Zusammenhang damit der Einzug solcher Forderungen (Factoringgeschäft)</p></div><div class="modal-footer"><button type="button" class="btn btn-default" data-dismiss="modal">Close description</button></div></div></div></div>
<div class="print-view-button-wrap">
<i class="fa fa-print" aria-hidden="true" role="presentation"></i>
<a class="company-db-print" href="https://www.fma.gv.at/wp-content/plugins/dw-fma/print.php" data-print="eNqFVMtu20gQvO9XNHSSsY4t2vKLFoyVEUcJjFxcskAM-DYkm-RA5FCYGUobB_tcJwvkYiA_ESBA9GOpHpLx-gHkILDZ01NdVd2Uig_izy6exqNEOR6dqzgKieN45Lxl9qNzF0c4ntd0ydWaM7ZMkWQP49GdXkmE4yiaTPow1f5TH95oNhKeINu0xttP_du8BbpWfZlRNYc-Z_HonfFsa2WX7OlSmSXNF3J0CgTluWg6iGiChBw7ekXds9IpG8cZaUM9_CyxtH8xlPVJ09ejzJdMV1dzGudQmjU1-YZWtlnrjMmxXQPR7XQg0vQoHq3KxvBgyZ_TQzqaQPjB6VmnHpbk6h-JJvGov8K10pXEB3h5z1XGG3Zs_vqf0D3le182nDjtuw5QuNls9vSzOnRJdNrDnxwe_-qd2uA2Znc2jaZTOPVvcK63xnXOHePWrJxe_PhKEc0Tt4fHLX6XN4vZPvKz1cVrDWOML7hgY1RZM8GhGqOnhQiwdNda-sh2oyrfmoIayanK0ZU2lSqYxn1gCnZpub3P_c45_aFah7dK8_YLKJWUsaNgQ4KJsBGRnt3ebH918Zzg4SOC6IcTgAGS1o0JxJaWM-3XbL3d3qM5tSajDFoWvNnel1aoDrWZshWXqBlfh1sPRF9uP33a_lq1eUD7Oy05xUJJsxtOS8eV2cV6uYRdY8L3XCIcXmu3bIzX_IvGUEzj_ux3HE6ecnirgF-RMGENwdxNohsWfVwwvpFmtfbdFpPaQbGrtvcm0-gF-bcKHprCocZ3XFx4rbGfr4Kej6pq_aMpvszs9On2bP9L2HarI1Ivt99sAQyVA22XFsoqsaKbEFxccl7oQ08u-4QoxwVSwb1dFORcMAsubr8JblPXXCIWG7jKZZPCEooLMtmKtetRKgX2nsZDv98ZHB0_WzKzHOb8prFZGB6AYVwi3SjLBsucD8ngPcRDzQOH3Y72gx2y9B-008vGkVxcwJeSWG0K0bFk7aG2wmTso34_vktLREF5gLwelt1pKRcHupnpmm5bp6SwVMhmCrMdWt21xYsNxm9U6hvh8dgj_IX8BMl1G1c,">Print view <span class="sr-only">(Opens in new tab)</span></a>
</div>
</div>

42
tests/data/at/ent_001.json Normal file
View File

@ -0,0 +1,42 @@
{
"name": "Intermarket Bank AG",
"address": "Am Belvedere 1, 1100 Wien, Austria",
"phone": "+43 5010028900",
"email": "Meldewesen@Intermarket.at",
"website": "www.intermarket.at",
"bankIdentificationNumber": "73600",
"commercialRegisterNumber": "94144y",
"categories": [
"Banks - Banks licensed in Austria",
"Banks - Austrian Banks in the EEA (freedom to provide services)"
],
"permissions": [
{
"heading": "§ 1 Abs. 1 Z 1 BWG",
"body": ["Die Entgegennahme fremder Gelder zur Verwaltung oder als Einlage (Einlagengeschäft); ausschließlich des Interbankenmarktes."]
},
{
"heading": "§ 1 Abs. 1 Z 3 BWG",
"body": ["Der Abschluss von Geldkreditverträgen und die Gewährung von Gelddarlehen (Kreditgeschäft)"]
},
{
"heading": "§ 1 Abs. 1 Z 4 BWG",
"body": ["Der Kauf von Schecks und Wechseln, insbesondere die Diskontierung von Wechseln (Diskontgeschäft)"]
},
{
"heading": "§ 1 Abs. 1 Z 7 BWG",
"body": [
"Der Handel auf eigene oder fremde Rechnung mit",
"a) ausländischen Zahlungsmitteln (Devisen- und Valutengeschäft);"
]
},
{
"heading": "§ 1 Abs. 1 Z 8 BWG",
"body": ["Die Übernahme von Bürgschaften, Garantien und sonstigen Haftungen für andere, sofern die übernommene Verpflichtung auf Geldleistungen lautet (Garantiegeschäft)"]
},
{
"heading": "§ 1 Abs. 1 Z 16 BWG",
"body": ["Der Ankauf von Forderungen aus Warenlieferungen oder Dienstleistungen, die Übernahme des Risikos der Einbringlichkeit solcher Forderungen ¿ ausgenommen die Kreditversicherung - und im Zusammenhang damit der Einzug solcher Forderungen (Factoringgeschäft)"]
}
]
}

21
tests/data/at/ent_002.html Normal file
View File

@ -0,0 +1,21 @@
<h3>
<span class="sr-only">Bitpanda Payments GmbH Wien</span>
<a href="#" class="company-details-toggle" rel="nofollow" title="Show Details" role="presentation" aria-hidden="true">
<span class="glyphicon indicator glyphicon-chevron-down" aria-hidden="true"></span>
Bitpanda Payments GmbH </a>
</h3>
<div class="company-details">
<ul>
<li><strong>Category:</strong> Payment institutions - Payment Institutions licensed in Austria<br></li><li><strong>Address:</strong> Jakov-Lind-Straße 2&nbsp;|&nbsp;1020&nbsp;Wien | Austria</li>
</ul>
<p class="toggle-document-description-wrap"><i class="fa fa-eye" aria-hidden="true"></i> <a href="#" data-toggle="modal" data-target="#document-description-5cc84f1a56622">Show licenses</a></p><div class="modal fade document-description" id="document-description-5cc84f1a56622" role="dialog" tabindex="-1" aria-labelledby="modal-label-5cc84f1a56622"><div class="modal-dialog modal-md" role="document"><div class="modal-content"><div class="modal-header"><h6 class="modal-title" id="modal-label-5cc84f1a56622"><span class="sr-only">Description for:</span> Licenses for&nbsp;Bitpanda Payments GmbH - Wien</h6></div><div class="modal-body"><h4>§ 1 Abs. 2 Z 3 ZaDiG 2018 - Zahlungsgeschäft</h4><p>Ausführung von Zahlungsvorgängen einschließlich des Transfers von Geldbeträgen auf ein Zahlungskonto beim Zahlungsdienstleister des Zahlungsdienstnutzers oder bei einem anderen Zahlungsdienstleister (Zahlungsgeschäft) a) Ausführung von Lastschriften einschließlich einmaliger Lastschriften (Lastschriftgeschäft); b) Ausführung von Zahlungsvorgängen mittels einer Zahlungskarte oder eines ähnlichen Instruments (Zahlungskartengeschäft); c) Ausführung von Überweisungen einschließlich Daueraufträgen (Überweisungsgeschäft); Die Konzession ist jedoch dahingehend eingeschränkt, dass die angebotenen Zahlungsdienste lediglich von Kunden der Bitpanda GmbH (FN 423018k) in Anspruch genommen werden dürfen;</p><h4>§ 1 Abs. 2 Z 6 ZaDiG 2018 - Finanztransfergeschäft</h4><p>Dienste, bei denen ohne Einrichtung eines Zahlungskontos auf den Namen des Zahlers oder des Zahlungsempfängers ein Geldbetrag eines Zahlers nur zum Transfer eines entsprechenden Betrags an den Zahlungsempfänger oder an einen anderen, im Namen des Zahlungsempfängers handelnden Zahlungsdienstleister entgegengenommen wird oder bei denen der Geldbetrag im Namen des Zahlungsempfängers entgegengenommen und diesem verfügbar gemacht wird (Finanztransfergeschäft) Die Konzession ist jedoch dahingehend eingeschränkt, dass die angebotenen Zahlungsdienste lediglich von Kunden der Bitpanda GmbH (FN 423018k) in Anspruch genommen werden dürfen;</p><h4>§ 1 Abs. 2 Z 7 ZaDiG 2018 - Zahlungsauslösedienst</h4><p>Dienste, die auf Antrag des Zahlungsdienstnutzers einen Zahlungsauftrag in Bezug auf ein bei einem anderen Zahlungsdienstleister geführtes Zahlungskonto auslösen (Zahlungsauslösedienste) Die Konzession ist jedoch dahingehend eingeschränkt, dass die angebotenen Zahlungsdienste lediglich von Kunden der Bitpanda GmbH (FN 423018k) in Anspruch genommen werden dürfen;</p></div><div class="modal-footer"><button type="button" class="btn btn-default" data-dismiss="modal">Close description</button></div></div></div></div>
<div class="print-view-button-wrap">
<i class="fa fa-print" aria-hidden="true" role="presentation"></i>
<a class="company-db-print" href="https://www.fma.gv.at/wp-content/plugins/dw-fma/print.php" data-print="eNrdVcFq20AQ_ZXBXCcbksZWQprIxpDgJm0TQqGFQm4rayxtLY3M7sohLv2TQi6G_kFPOVU_1tmVLNuRTXttb9qdmTdvZt6shO_5X7V_4rcCobHVF_6ZPZ_6LW0Uomn1te91_dZ7Mc3mh7eSwsOPRoniCcGztmO_tZAz-8UQva7XrT7H0jxWn58lkv18zbdZTkY9VqeLnHNIUbmRSNFl8_zWpTQzQaGAD-IxRTIartPgrbWeMYgwGGUlyum536p8QJI20uRGZqThcBUK7zavEzlG0hiyM1TpB4GCo2FVSyDH9osLri7GiuqLby59BaHtde_cYwKD-GT46wf04Fwi0K_Ag3s4hnsxktfgdXtnTOVexElOkY5Qj-NiOTGDIw4ZzIZMYVI8x4qNMM-odpxnKiqWFCEBclnjOJFYPHHmGELU8EkJ0hNU2gVdYxIGaFSxtP5cIp_YmBpqmpHJIECZ1lchD0SbBKU2qBzgtoVys7DgWchmjrR4mAIPBBXSHph2o8gOiA68rPBWaMN2JVwnZkdxfE5FXCIjxtt2bG8c1wn6EDQz7OhhKo3BRLs61LozQhksi7QGDcUyJkuDI6xoVF5Kr70VQJvpx830xfdcMNUDNyXfOb6RyFHxjFbjam_6603skUS4yWiBWrN0gbsMXzDMrARELBmceYYW3wUxHE3NAdu0Bp4MTyvCIGO-jYkhJBjKyNGxjG9yHiyB7UO9d3bdoH11B1wn3jFreNpxC0N6pnKOYuJZys2BB1QutHhWE6T-4Gg2bC7D6fYyXFxJErQwlYQbKzEqOR444YWOfhYTwhtJihkb2-dyXFxbCtdO-JbMnUhdNaVDreNNmWM6mzhpKKeJeoPEJrQ1Uq5gkaf1wlVmK4uZQqsUm_HShTIDcgSaWUoGwqnBrmi5SAfAO7nN9iW52LomFO7dOmYSYWRFuZqIVOF6ccv-2dNGiX_M2lwwZYFYTbETzFGx3KNAKFZBKnggZcr2nrF2_h8dv979qItcXFwnxU_-pzhSDRm7GliaF-Sav_-1LbWxRp2Uw7LyWuRR_a7_7XscoXuWzMtFgRVfWj9s2yXgPzsz_kX_Bj1RQTI,">Print view <span class="sr-only">(Opens in new tab)</span></a>
</div>
</div>

26
tests/data/at/ent_002.json Normal file
View File

@ -0,0 +1,26 @@
{
"name": "Bitpanda Payments GmbH",
"address": "Jakov-Lind-Straße 2, 1020 Wien, Austria",
"phone": "",
"email": "",
"website": "",
"bankIdentificationNumber": "",
"commercialRegisterNumber": "",
"categories": [
"Payment institutions - Payment Institutions licensed in Austria"
],
"permissions": [
{
"heading": "§ 1 Abs. 2 Z 3 ZaDiG 2018 - Zahlungsgeschäft",
"body": ["Ausführung von Zahlungsvorgängen einschließlich des Transfers von Geldbeträgen auf ein Zahlungskonto beim Zahlungsdienstleister des Zahlungsdienstnutzers oder bei einem anderen Zahlungsdienstleister (Zahlungsgeschäft) a) Ausführung von Lastschriften einschließlich einmaliger Lastschriften (Lastschriftgeschäft); b) Ausführung von Zahlungsvorgängen mittels einer Zahlungskarte oder eines ähnlichen Instruments (Zahlungskartengeschäft); c) Ausführung von Überweisungen einschließlich Daueraufträgen (Überweisungsgeschäft); Die Konzession ist jedoch dahingehend eingeschränkt, dass die angebotenen Zahlungsdienste lediglich von Kunden der Bitpanda GmbH (FN 423018k) in Anspruch genommen werden dürfen;"]
},
{
"heading": "§ 1 Abs. 2 Z 6 ZaDiG 2018 - Finanztransfergeschäft",
"body": ["Dienste, bei denen ohne Einrichtung eines Zahlungskontos auf den Namen des Zahlers oder des Zahlungsempfängers ein Geldbetrag eines Zahlers nur zum Transfer eines entsprechenden Betrags an den Zahlungsempfänger oder an einen anderen, im Namen des Zahlungsempfängers handelnden Zahlungsdienstleister entgegengenommen wird oder bei denen der Geldbetrag im Namen des Zahlungsempfängers entgegengenommen und diesem verfügbar gemacht wird (Finanztransfergeschäft) Die Konzession ist jedoch dahingehend eingeschränkt, dass die angebotenen Zahlungsdienste lediglich von Kunden der Bitpanda GmbH (FN 423018k) in Anspruch genommen werden dürfen;"]
},
{
"heading": "§ 1 Abs. 2 Z 7 ZaDiG 2018 - Zahlungsauslösedienst",
"body": ["Dienste, die auf Antrag des Zahlungsdienstnutzers einen Zahlungsauftrag in Bezug auf ein bei einem anderen Zahlungsdienstleister geführtes Zahlungskonto auslösen (Zahlungsauslösedienste) Die Konzession ist jedoch dahingehend eingeschränkt, dass die angebotenen Zahlungsdienste lediglich von Kunden der Bitpanda GmbH (FN 423018k) in Anspruch genommen werden dürfen;"]
}
]
}

88
tests/data/at/ent_003.html Normal file
View File

@ -0,0 +1,88 @@
<h3>
<span class="sr-only">DIMOCO Europe GmbH Brunn am Gebirge</span>
<a href="#" class="company-details-toggle" rel="nofollow" title="Show Details" role="presentation"
aria-hidden="true">
<span class="glyphicon indicator glyphicon-chevron-down" aria-hidden="true"></span>
DIMOCO Europe GmbH </a>
</h3>
<div class="company-details">
<ul>
<li><strong>Category:</strong> Payment institutions - Payment Institutions licensed in Austria<br>Payment
institutions - Austrian Payment Institutions in EEA (freedom to provide services)<br></li>
<li><strong>Address:</strong> Europaring F15/302&nbsp;|&nbsp;2345&nbsp;Brunn am Gebirge | Austria</li>
<li><strong>Legal Identifiers:</strong>
<ul>
<li><strong>Commercial register number:</strong> 199901y</li>
</ul>
</li>
</ul>
<p class="toggle-document-description-wrap"><i class="fa fa-eye" aria-hidden="true"></i> <a href="#"
data-toggle="modal"
data-target="#document-description-5cf8e5955d032">Show
licenses</a></p>
<div class="modal fade document-description" id="document-description-5cf8e5955d032" role="dialog" tabindex="-1"
aria-labelledby="modal-label-5cf8e5955d032" style="display: none;">
<div class="modal-dialog modal-md" role="document">
<div class="modal-content">
<div class="modal-header"><h6 class="modal-title" id="modal-label-5cf8e5955d032"><span class="sr-only">Description for:</span>
Licenses for&nbsp;DIMOCO Europe GmbH - Brunn am Gebirge</h6></div>
<div class="modal-body">
<h4>§ 1 Abs. 2 Z 3 ZaDiG 2018 - Zahlungsgeschäft</h4>
<p>Ausführung von Zahlungsvorgängen einschließlich des Transfers von Geldbeträgen auf ein
Zahlungskonto beim Zahlungsdienstleister des Zahlungsdienstnutzers oder bei einem anderen
Zahlungsdienstleister (Zahlungsgeschäft) a) Ausführung von Lastschriften einschließlich
einmaliger Lastschriften (Lastschriftgeschäft); b) Ausführung von Zahlungsvorgängen mittels
einer Zahlungskarte oder eines ähnlichen Instruments (Zahlungskartengeschäft); c) Ausführung von
Überweisungen einschließlich Daueraufträgen (Überweisungsgeschäft);</p>
<h4>§ 1 Abs. 2 Z 5 ZaDiG
2018 - Issuing oder Acquiring</h4>
<p>Annahme und Abrechnung von Zahlungsvorgängen (Acquiring);</p>
<h4>Zahlungsdienste gemäß Anhang RI
2007/64/EG im Rahmen der Dienstleistungsfreiheit in Belgien</h4>
<p> 3. Zahlungsgeschäft</p>
<h4>Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der
Dienstleistungsfreiheit in Belgien</h4>
<p> 5. Zahlungsinstrumentegeschäft</p>
<h4>Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der
Dienstleistungsfreiheit in Belgien (Zusätze/Einschränkungen)</h4>
<p>3. Ausführung von Zahlungsvorgängen einschließlich des Transfers von Geldbeträgen auf ein
Zahlungskonto beim Zahlungsdienstleister des Nutzers oder bei einem anderen
Zahlungsdienstleister:<br>a) Ausführung von Lastschriften einschließlich einmaliger
Lastschriften<br>b) Ausführung von Zahlungsvorgängen mittels einer Zahlungskarte oder eines
ähnli-chen Instruments<br>c) Ausführung von Überweisungen einschließlich Daueraufträgen<br>5.
Annahme und Abrechnung ("acquiring") von Zahlungsinstrumenten (ohne die Gewäh-rung von Krediten
entsprechend Artikel 18(4) der RL (EU) 2015/2366)<br></p>
<h4>Zahlungsdienste gemäß Anhang RI
2007/64/EG im Rahmen der Dienstleistungsfreiheit in Bulgarien</h4>
<p> 3. Zahlungsgeschäft</p>
<h4>Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der
Dienstleistungsfreiheit in Bulgarien</h4>
<p> 5. Zahlungsinstrumentegeschäft</p>
<h4>Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der
Dienstleistungsfreiheit in Bulgarien (Zusätze/Einschränkungen)</h4>
<p>3. Ausführung von Zahlungsvorgängen einschließlich des Transfers von Geldbeträgen auf ein
Zahlungskonto beim Zahlungsdienstleister des Nutzers oder bei einem anderen
Zahlungsdienstleister:<br>a) Ausführung von Lastschriften einschließlich einmaliger
Lastschriften<br>b) Ausführung von Zahlungsvorgängen mittels einer Zahlungskarte oder eines
ähnli-chen Instruments<br>c) Ausführung von Überweisungen einschließlich Daueraufträgen<br>5.
Annahme und Abrechnung ("acquiring") von Zahlungsinstrumenten (ohne die Gewäh-rung von Krediten
entsprechend Artikel 18(4) der RL (EU) 2015/2366)<br></p>
</div>
<div class="modal-footer">
<button type="button" class="btn btn-default" data-dismiss="modal">Close description</button>
</div>
</div>
</div>
</div>
<div class="print-view-button-wrap">
<i class="fa fa-print" aria-hidden="true" role="presentation"></i>
<a class="company-db-print" href="https://www.fma.gv.at/wp-content/plugins/dw-fma/print.php"
data-print="eNrtnc1u2zgQgM99C8JcJ_vQOo6TNHWCXDDpxs0G_UXaXnKj5IlERKK8JJVsuuibLNBLgH2JPa1fbIey_BNL2cUu2k5BzCmWSA4_kvONlThB5Gh79Jsd7Yw6kbTQOZCjfX-9N-pYZ1wwXFznwI4G-6POuDTFVBqlE_FisNsfbm37luGo80lN_SsMsD3c2a1fxsrdViMxznNTai1kLk4hUiYBf_8pdilK7cxtfXVcXOJ0StbDtcxhMfHJ2eu3P70V1fwgTvPoZ9-CDbF0kBSmngdHvZO3OWhcJ5S2TrnSqUJb8Vgsbp-t385UDNrCBDuLeu7DyIj-0QNB6j66PRoGGY-PRfcSd2xS5MIVYmqKazUBYcFcXONcXLY3D19vWqRi_2pr1KlvxEbXOzF49uzZ1lwwV_W5WmUNaqt-g-EWbuhhunP01x9iII4j-0RsiwsxFBfyRJ2K7a3BPsJeyDQrdWITsHE6u7t0h30ccjg9wlVcXM7-TPFAEnFd6GXH68IkszudgBaAC4_TTMHsC06diglY8cFIbS_B2GrQKWSTCJyZ3fn-srz0Y5ahrgqNq49A5ctbE4UrcBko68BUAe-36NJ98sGLCTbjSB8PciE1XoN-IEy3sciekD2xucJX0jpsN-rStSwOr3OZqQTj3e_YXbtcXE1wIKLmDC17mCvnILPVOsxqZ6RxMF-kb7Bidpdqj4EjfDaZ0lwnll2trBqg16ePm9PPfo_A3OCmlK3Hd1wiSzB4Rovj6q73X9u7g8P-9KiZWLv3E-vM2tJXgGoRx_EvpfIFYZldWss0B1HqCUYwEKf6Hzapuxy_mvz-UYNIIJ_dzb6IY51KjHR-hiBbT_t7O_3xqcAMO_fzaeFpTlbZ4SOgiSoF5T0WzyFLsHWBKcTwSZtcIt8VYXeFoJZnD9-XBjOttLM7lK8_rtIGc0RfVXnUW5AOnzz6UYrGm_9RJkaHz8_7R7L36OvUhSpa1Hv0tWvA481cIlBNFDdcJ_pvtldRdvEE28XsduRCwU7v3jrWUhKzpEg1CNxWPMQb5H28pHlpYKKq_UPkqY8Lfgbj1BVkYrDf3elVqXj-SnTHH3u-iuz2t4d7e72K7BtcJ3mZJfi8Qux9E4LW_AUPu8_uh-v-CR4E5NJcXFG63wJB6v6Sh91n9wN2H0p_cBnmB6n9bRi0_q-IuAJwBQi3AozxPrH9DQRS82satp6tD9f6F0prau2bDKTeL3BYfBY_YPEx_67wOk5J1W-hoJV_CcT6s_7h6n9qVAVE_d7fzkFaAtaRuAhwEQi5CBSzL5FRTmpN_JHfgyjEpeAeFVcDrgbhVoMzQ_0wsElA6v4chpVn5QNW3pIrb38k5S0rz8oHrrzDE6R91m8g0Eo_p2Hr2fpwrX9pCumItW8ykHq_wGHxWfxwxX8FjvzXepoMpOIvcFh8Fj9g8f2HWK6KqEjf9h9cMKEtAetMXFwHuA6EXFwHHO4GcQXYQCB2v6Jh69n6gK0vf4U8Kk1C6n0Tgtb8BQ-7z-6H6_5rmTlJ6f0GXDCp8xUL-86-h-v7GwXYw_9ECyitb8UgdX-NiCtcMFeAgCtAgRuV0H6T32Sgdb_GYfFZ_HDFf1dktNZvXDCQKl-xsO_se8i-G1cmMqNVfpOB2Po5DovP4ocr_nmJUaj_Rq8FglT9JQ-7z-5Tut_7tnn-Pk5vYELrfpOBVP0FDpvP5of7rv8-K27kFShS8RsMtOLXOCw-ix-2-ED9uN8CQa4-8OM-ux-4-1NJbv4mAq33cxq2nq0P1_oPuDNIpPwXcQ7TMsoU6f_h-Bcg0orQysb1getDuPXho06kIX0o2CQgrVwwcxhWnpUPV_mL2ynQKr9JQKr8HIaVZ-Wple8cfP4b_1Av1g,,">Print
view <span class="sr-only">(Opens in new tab)</span></a>
</div>
</div>

59
tests/data/at/ent_003.json Normal file
View File

@ -0,0 +1,59 @@
{
"name": "DIMOCO Europe GmbH",
"address": "Europaring F15/302, 2345 Brunn am Gebirge, Austria",
"phone": "",
"email": "",
"website": "",
"bankIdentificationNumber": "",
"commercialRegisterNumber": "199901y",
"categories": [
"Payment institutions - Payment Institutions licensed in Austria",
"Payment institutions - Austrian Payment Institutions in EEA (freedom to provide services)"
],
"permissions": [
{
"heading": "§ 1 Abs. 2 Z 3 ZaDiG 2018 - Zahlungsgeschäft",
"body": ["Ausführung von Zahlungsvorgängen einschließlich des Transfers von Geldbeträgen auf ein Zahlungskonto beim Zahlungsdienstleister des Zahlungsdienstnutzers oder bei einem anderen Zahlungsdienstleister (Zahlungsgeschäft) a) Ausführung von Lastschriften einschließlich einmaliger Lastschriften (Lastschriftgeschäft); b) Ausführung von Zahlungsvorgängen mittels einer Zahlungskarte oder eines ähnlichen Instruments (Zahlungskartengeschäft); c) Ausführung von Überweisungen einschließlich Daueraufträgen (Überweisungsgeschäft);"]
},
{
"heading": "§ 1 Abs. 2 Z 5 ZaDiG 2018 - Issuing oder Acquiring",
"body": ["Annahme und Abrechnung von Zahlungsvorgängen (Acquiring);"]
},
{
"heading": "Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der Dienstleistungsfreiheit in Belgien",
"body": ["3. Zahlungsgeschäft"]
},
{
"heading": "Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der Dienstleistungsfreiheit in Belgien",
"body": ["5. Zahlungsinstrumentegeschäft"]
},
{
"heading": "Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der Dienstleistungsfreiheit in Belgien (Zusätze/Einschränkungen)",
"body": [
"3. Ausführung von Zahlungsvorgängen einschließlich des Transfers von Geldbeträgen auf ein Zahlungskonto beim Zahlungsdienstleister des Nutzers oder bei einem anderen Zahlungsdienstleister:",
"a) Ausführung von Lastschriften einschließlich einmaliger Lastschriften",
"b) Ausführung von Zahlungsvorgängen mittels einer Zahlungskarte oder eines ähnli-chen Instruments",
"c) Ausführung von Überweisungen einschließlich Daueraufträgen",
"5. Annahme und Abrechnung (\"acquiring\") von Zahlungsinstrumenten (ohne die Gewäh-rung von Krediten entsprechend Artikel 18(4) der RL (EU) 2015/2366)"
]
},
{
"heading": "Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der Dienstleistungsfreiheit in Bulgarien",
"body": ["3. Zahlungsgeschäft"]
},
{
"heading": "Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der Dienstleistungsfreiheit in Bulgarien",
"body": ["5. Zahlungsinstrumentegeschäft"]
},
{
"heading": "Zahlungsdienste gemäß Anhang RI 2007/64/EG im Rahmen der Dienstleistungsfreiheit in Bulgarien (Zusätze/Einschränkungen)",
"body": [
"3. Ausführung von Zahlungsvorgängen einschließlich des Transfers von Geldbeträgen auf ein Zahlungskonto beim Zahlungsdienstleister des Nutzers oder bei einem anderen Zahlungsdienstleister:",
"a) Ausführung von Lastschriften einschließlich einmaliger Lastschriften",
"b) Ausführung von Zahlungsvorgängen mittels einer Zahlungskarte oder eines ähnli-chen Instruments",
"c) Ausführung von Überweisungen einschließlich Daueraufträgen",
"5. Annahme und Abrechnung (\"acquiring\") von Zahlungsinstrumenten (ohne die Gewäh-rung von Krediten entsprechend Artikel 18(4) der RL (EU) 2015/2366)"
]
}
]
}

11
tests/data/be/ci_001_fullDetails.json Normal file
View File

@ -0,0 +1,11 @@
{
"name": "ABN AMRO Private Banking Belgium",
"companyType": "Société anonyme",
"addressOne": "Kortrijksesteenweg 302",
"addressTwo": "9000 Gent",
"addressThree": null,
"uniqueId": "0415.835.337",
"dateOfListing": null,
"docLink": null,
"normalisedDocLink": null
}

11
tests/data/be/ci_001_mainDetails.json Normal file
View File

@ -0,0 +1,11 @@
{
"name": "ABN AMRO Private Banking Belgium",
"companyType": "Société anonyme",
"addressOne": "Kortrijksesteenweg 302",
"addressTwo": "9000 Gent",
"addressThree": null,
"uniqueId": "0415.835.337",
"dateOfListing": null,
"docLink": null,
"normalisedDocLink": null
}

1004
tests/data/be/ci_fullpage_001.html Normal file
View File

File diff suppressed because one or more lines are too long

14
tests/data/be/em_001_fullDetails.json Normal file
View File

@ -0,0 +1,14 @@
{
"name": "Buy Way Personal Finance",
"companyType": "Société anonyme",
"addressOne": "Rue de l'Evêque 26",
"addressTwo": "1000 Bruxelles",
"addressThree": null,
"uniqueId": "0400.282.277",
"dateOfListing": "07-05-2013",
"docLink": "http://www.nbb.be/DOC/CP/ENG/psd/Buy%20Way.docx?t=19418",
"normalisedDocLink": "http://www.nbb.be/DOC/CP/ENG/psd/Buy%20Way.docx?t=19418",
"activitesGenerales": [ "A", "B" ],
"servicesDePaiement": [ "1", "2", "3", "4", "5", "6", "7", "8" ],
"autresServicesLies": [ "X" ]
}

11
tests/data/be/em_001_mainDetails.json Normal file
View File

@ -0,0 +1,11 @@
{
"name": "Ingenico Financial Solutions",
"companyType": "Société anonyme",
"addressOne": "Leonardo Da Vincilaan 3",
"addressTwo": "Corporate Village, Bayreuth Building",
"addressThree": "1930 Zaventem",
"uniqueId": "0886.476.763",
"dateOfListing": "30-11-2012",
"docLink": "http://www.nbb.be/DOC/CP/ENG/psd/Ingenico.docx?t=19418",
"normalisedDocLink": "http://www.nbb.be/DOC/CP/ENG/psd/Ingenico.docx?t=19418"
}

509
tests/data/be/em_fullpage_001.html Normal file
View File

File diff suppressed because one or more lines are too long

10
tests/data/be/ps_001.html Normal file
View File

@ -0,0 +1,10 @@
<td><strong>Airplus International</strong><br>
<em>Société anonyme</em><br>
Boulevard de l'Impératrice 66 <br>
1000 Bruxelles<br>
<br>
Numéro d'identification unique : &nbsp;0883.523.807<br>
Date de l'inscription à la liste : &nbsp;26-04-2011<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="http://www.nbb.be/DOC/CP/ENG/psd/BCCC.docx?t=1949" target="_blank">voyez la liste en annexe</a></td>
<td>2 3 4 5</td>
<td>X</td>

13
tests/data/be/ps_001.json Normal file
View File

@ -0,0 +1,13 @@
{
"name": "Airplus International",
"companyType": "Société anonyme",
"addressOne": "Boulevard de l'Impératrice 66",
"addressTwo": "1000 Bruxelles",
"uniqueId": "0883.523.807",
"dateOfListing": "26-04-2011",
"paymentServices": [
"2", "3", "4", "5"
],
"otherRelatedServices": "X",
"docLink": "http://www.nbb.be/DOC/CP/ENG/psd/BCCC.docx?t=1949"
}

13
tests/data/be/ps_001_fullDetails.json Normal file
View File

@ -0,0 +1,13 @@
{
"name": "Airplus International",
"companyType": "Société anonyme",
"addressOne": "Boulevard de l'Impératrice 66",
"addressTwo": "1000 Bruxelles",
"addressThree": null,
"uniqueId": "0883.523.807",
"dateOfListing": "26-04-2011",
"docLink": "http://www.nbb.be/DOC/CP/ENG/psd/BCCC.docx?t=19417",
"normalisedDocLink": "http://www.nbb.be/DOC/CP/ENG/psd/BCCC.docx?t=19417",
"servicesDePaiement": [ "2", "3", "4", "5" ],
"autresServicesLies": [ "X" ]
}

11
tests/data/be/ps_001_mainDetails.json Normal file
View File

@ -0,0 +1,11 @@
{
"name": "Airplus International",
"companyType": "Société anonyme",
"addressOne": "Boulevard de l'Impératrice 66",
"addressTwo": "1000 Bruxelles",
"addressThree": null,
"uniqueId": "0883.523.807",
"dateOfListing": "26-04-2011",
"docLink": "http://www.nbb.be/DOC/CP/ENG/psd/BCCC.docx?t=19417",
"normalisedDocLink": "http://www.nbb.be/DOC/CP/ENG/psd/BCCC.docx?t=19417"
}

751
tests/data/be/ps_fullpage_001.html Normal file
View File

File diff suppressed because one or more lines are too long

500
tests/data/be/ps_index_001.html Normal file
View File

@ -0,0 +1,500 @@
<div class="field-name-field-page-intro">
<p>Etablissements de paiement agréés en Belgique</p>
</div>
<div class="field-name-table-of-contents">
<h4>
<a aria-label="Toggle table of contents" class="icon-table-of-contents" data-toggle="collapse" href="#table-of-contents">Contenu</a>
</h4>
<nav class="collapse in" id="table-of-contents">
<ul>
<li>
<a href="#bm_Header_0">Etablissements de paiement de droit belge</a>
</li>
</ul>
</nav>
</div>
<div class="description">
Article 8 §1, 1° de la loi du 11 mars 2018 relative au statut et au contrôle des établissements de paiement et des établissements de monnaie électronique, à l'accès à l'activité de prestataire de services de paiement, et à l'activité d'émission de monnaie électronique, et à l'accès aux systèmes de paiement
</div>
<div class="legend">
<h3>(*) Services de paiement</h3>
<ol>
<li>Les services permettant de verser des espèces sur un compte de paiement et toutes les opérations qu’exige la gestion d’un compte de paiement;</li>
<li>Les services permettant de retirer des espèces d’un compte de paiement et toutes les opérations qu’exige la gestion d’un compte de paiement;</li>
<li>L’exécution d’opérations de paiement, y compris les transferts de fonds sur un compte de paiement auprès du prestataire de services de paiement de l’utilisateur ou auprès d’un autre prestataire de services de paiement :
<ol style="list-style-type:lower-alpha">
<li>l’exécution de prélèvements, y compris de prélèvements autorisés unitairement,</li>
<li>l’exécution d’opérations de paiement à l’aide d’une carte de paiement ou d’un dispositif similaire,</li>
<li>l’exécution de virements, y compris d’ordres permanents;</li>
</ol>
</li>
<li>L’exécution d’opérations de paiement dans le cadre desquelles les fonds sont couverts par une ligne de crédit accordée à l’utilisateur de services de paiement :
<ol style="list-style-type:lower-alpha">
<li>l’exécution de prélèvements, y compris de prélèvements autorisés unitairement,</li>
<li>l’exécution d’opérations de paiement à l’aide d’une carte de paiement ou d’un dispositif similaire,</li>
<li>l’exécution de virements, y compris d’ordres permanents;</li>
</ol>
</li>
<li>L’émission d’instruments de paiement et/ou l’acquisition d’opérations de paiement;</li>
<li>Les transmissions de fonds;</li>
<li>Les services d’initiation de paiement;</li>
<li>Les services d’information sur les comptes;</li>
</ol>
<h3>(**) Autres services liés</h3>
<ol style="list-style-type:upper-alpha">
<li value="24">Octroi de crédits liés aux services de paiement visés aux points 4, 5 ou 7 des services de paiement visés sous (*)</li>
</ol>
</div>
<h2>
Situation au&nbsp;08-04-2019
</h2>
<a href="#bm_Changes12M">Modifications de la liste au cours des douze derniers mois</a>
<ul class="List1">
<li>
<h3 id="bm_Header_0">
Etablissements de paiement de droit belge
</h3>
<div class="number-of-records">
Nombre total d'établissements : &nbsp;24
</div>
<div class="table-responsive">
<table class="table">
<thead>
<tr>
<th></th>
<th>Services de paiement<div class="stars">(*)</div></th>
<th>Autres services liés<div class="stars">(**)</div></th>
</tr>
</thead>
<tbody>
<tr>
<td><strong>Airplus International</strong><br>
<em>Société anonyme</em><br>
Boulevard de l'Impératrice 66 <br>
1000 Bruxelles<br>
<br>
Numéro d'identification unique : &nbsp;0883.523.807<br>
Date de l'inscription à la liste : &nbsp;26-04-2011<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="http://www.nbb.be/DOC/CP/ENG/psd/BCCC.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
<td>2 3 4 5</td>
<td>X</td>
</tr>
<tr>
<td><strong>Alpha Card</strong><br>
<em>Société coopérative à responsabilité limitée</em><br>
Boulevard du Souverain 100 <br>
1170 Watermael Boitsfort<br>
<br>
Numéro d'identification unique : &nbsp;0463.926.551<br>
Date de l'inscription à la liste : &nbsp;26-04-2011<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="http://www.nbb.be/DOC/CP/ENG/psd/Alpha%20Card.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
<td>2 3 4 5</td>
<td>X</td>
</tr>
<tr>
<td><strong>Alpha Card Merchant Services</strong><br>
<em>Société coopérative à responsabilité limitée</em><br>
Boulevard du Souverain 100 <br>
1170 Watermael Boitsfort<br>
<br>
Numéro d'identification unique : &nbsp;0475.933.171<br>
Date de l'inscription à la liste : &nbsp;26-04-2011<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="http://www.nbb.be/DOC/CP/ENG/psd/Alpha%20Card%20Merchant%20Services.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
<td>2 3 4 5</td>
<td>X</td>
</tr>
<tr>
<td><strong>Belmoney Transfert</strong><br>
<em>Société privée à responsabilité limitée</em><br>
Place Bara 28 <br>
1070 Bruxelles<br>
<br>
Numéro d'identification unique : &nbsp;0540.745.997<br>
Date de l'inscription à la liste : &nbsp;20-03-2018<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="http://www.nbb.be/DOC/CP/ENG/psd/Belmoney.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
<td>6</td>
<td></td>
</tr>
<tr>
<td><strong>Cashfree</strong><br>
<em>Société anonyme</em><br>
Mortelputstraat 49 <br>
9830 Sint-Martens-Latem<br>
<br>
Numéro d'identification unique : &nbsp;0663.774.859<br>
Date de l'inscription à la liste : &nbsp;11-07-2017</td>
<td>3 4 5</td>
<td></td>
</tr>
<tr>
<td><strong>Cofidis</strong><br>
<em>Société anonyme</em><br>
Chaussée de Lille 422A <br>
7501 Orcq<br>
<br>
Numéro d'identification unique : &nbsp;0400.359.283<br>
Date de l'inscription à la liste : &nbsp;07-05-2013</td>
<td>4 5 6</td>
<td>X</td>
</tr>
<tr>
<td><strong>Digiteal</strong><br>
<em>Société anonyme</em><br>
rue Emile Francqui 6 <br>
1435 Corbais<br>
<br>
Numéro d'identification unique : &nbsp;0630.675.588<br>
Date de l'inscription à la liste : &nbsp;31-10-2017<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="http://www.nbb.be/DOC/CP/ENG/psd/Teal%20IT.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
<td>3 7 8</td>
<td></td>
</tr>
<tr>
<td><strong>Ebury Partners Belgium</strong><br>
<em>Société anonyme</em><br>
Boulevard du Régent 37 <br>
1000 Bruxelles<br>
<br>
Numéro d'identification unique : &nbsp;0681.746.187<br>
Date de l'inscription à la liste : &nbsp;31-10-2017<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="file:///L:/PRXNWEBP/DOC/CP/eng/psd/Ebury.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
<td>3 4 6</td>
<td></td>
</tr>
<tr>
<td><strong>eDebex</strong><br>
<em>Société anonyme</em><br>
Rue Jules Cockx 8-10 <br>
1160 Bruxelles<br>
<br>
Numéro d'identification unique : &nbsp;0502.697.352<br>
Date de l'inscription à la liste : &nbsp;02-02-2016<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="http://www.nbb.be/DOC/CP/ENG/psd/Edebex.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
<td>3</td>
<td></td>
</tr>
<tr>
<td><strong>EPBF</strong><br>
<em>Société anonyme</em><br>
Chaussée de la Hulpe 181 <br>
1170 Watermael-Boitsfort<br>
<br>
Numéro d'identification unique : &nbsp;0837.808.497<br>
Date de l'inscription à la liste : &nbsp;27-06-2011<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="http://www.nbb.be/DOC/CP/ENG/psd/EPBF.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
<td>3 4 5</td>
<td>X</td>
</tr>
<tr>
<td><strong>Gold Commodities Forex</strong><br>
<em>Société anonyme</em><br>
Winderickxplein 3 bte 2<br>
1652 Beersel<br>
<br>
Numéro d'identification unique : &nbsp;0832.602.171<br>
Date de l'inscription à la liste : &nbsp;27-06-2011<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="http://www.nbb.be/DOC/CP/ENG/psd/GCF.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
<td>3 6</td>
<td></td>
</tr>
<tr>
<td><strong>HomeSend</strong><br>
<em>Société coopérative à responsabilité limitée</em><br>
Rue des Colonies 56 <br>
1000 Bruxelles<br>
<br>
Numéro d'identification unique : &nbsp;0549.987.921<br>
Date de l'inscription à la liste : &nbsp;09-02-2016<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="http://www.nbb.be/DOC/CP/ENG/psd/HomeSend.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
<td>3 4</td>
<td></td>
</tr>
<tr>
<td><strong>iBanFirst</strong><br>
<em>Société anonyme</em><br>
Avenue Louise 350 <br>
1050 Bruxelles<br>
<br>
Numéro d'identification unique : &nbsp;0849.872.824<br>
Date de l'inscription à la liste : &nbsp;22-05-2013<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="http://www.nbb.be/DOC/CP/ENG/psd/FX4Biz.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
<td>3 4 7 8</td>
<td></td>
</tr>
<tr>
<td><strong>Isabel</strong><br>
<em>Société anonyme</em><br>
Boulevard de l'Impératrice 13-15 <br>
1000 Bruxelles<br>
<br>
Numéro d'identification unique : &nbsp;0455.530.509<br>
Date de l'inscription à la liste : &nbsp;19-02-2019<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="file:///L:/PRXNWEBP/DOC/CP/eng/psd/Isabel.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
<td>7 8</td>
<td></td>
</tr>
<tr>
<td><strong>Let's Didid</strong><br>
<em>Société anonyme</em><br>
Montagne du Parc 3 <br>
1000 Bruxelles<br>
<br>
Numéro d'identification unique : &nbsp;0450.355.261<br>
Date de l'inscription à la liste : &nbsp;19-02-2019</td>
<td>7 8</td>
<td></td>
</tr>
<tr>
<td><strong>Money International</strong><br>
<em>Société anonyme</em><br>
Chaussée de Gand 100 <br>
1080 Bruxelles<br>
<br>
Numéro d'identification unique : &nbsp;0475.445.104<br>
Date de l'inscription à la liste : &nbsp;26-03-2019</td>
<td>6</td>
<td></td>
</tr>
<tr>
<td><strong>MoneyGram International</strong><br>
<em>Société privée à responsabilité limitée</em><br>
rue Joseph Stevens 7 <br>
1000 Bruxelles<br>
<br>
Numéro d'identification unique : &nbsp;0671.690.653<br>
Date de l'inscription à la liste : &nbsp;19-12-2017<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="http://www.nbb.be/DOC/CP/ENG/psd/MoneyGram.xlsx?t=19411" target="_blank">voyez la liste en annexe</a></td>
<td>6</td>
<td></td>
</tr>
<tr>
<td><strong>Moneytrans Payment Services</strong><br>
<em>Société anonyme</em><br>
Boulevard de Waterloo 77 <br>
1000 Bruxelles<br>
<br>
Numéro d'identification unique : &nbsp;0449.356.557<br>
Date de l'inscription à la liste : &nbsp;26-04-2011<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="http://www.nbb.be/DOC/CP/ENG/psd/change_center_sprl.xlsx?t=19411" target="_blank">voyez la liste en annexe</a></td>
<td>3 6</td>
<td></td>
</tr>
<tr>
<td><strong>Oonex</strong><br>
<em>Société anonyme</em><br>
Avenue Louise 367 <br>
1050 Bruxelles<br>
<br>
Numéro d'identification unique : &nbsp;0849.185.510<br>
Date de l'inscription à la liste : &nbsp;23-08-2016<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="http://www.nbb.be/DOC/CP/ENG/psd/Oonex.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
<td>3 5</td>
<td></td>
</tr>
<tr>
<td><strong>PAY-NXT </strong><br>
<em>Société anonyme</em><br>
avenue Reine Astrid 92A <br>
1310 La Hulpe<br>
<br>
Numéro d'identification unique : &nbsp;0649.860.804<br>
Date de l'inscription à la liste : &nbsp;11-10-2016<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="http://www.nbb.be/DOC/CP/ENG/psd/Pay%20Nxt.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
<td>3 5</td>
<td></td>
</tr>
<tr>
<td><strong>Transferwise Europe</strong><br>
<em>Société anonyme</em><br>
Square de Meeûs 38 bte 40<br>
1000 Bruxelles<br>
<br>
Numéro d'identification unique : &nbsp;0713.629.988<br>
Date de l'inscription à la liste : &nbsp;19-03-2019<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="file:///L:/PRXNWEBP/DOC/CP/eng/psd/TransferWise%20Europe.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
<td>3 5 6</td>
<td></td>
</tr>
<tr>
<td><strong>Travelex</strong><br>
<em>Société anonyme</em><br>
Steendam 108 <br>
9000 Gent<br>
<br>
Numéro d'identification unique : &nbsp;0451.299.329<br>
Date de l'inscription à la liste : &nbsp;26-04-2011<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="http://www.nbb.be/DOC/CP/ENG/psd/Travelex.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
<td>6</td>
<td></td>
</tr>
<tr>
<td><strong>Worldline</strong><br>
<em>Société anonyme</em><br>
Chaussée de Haecht 1442 <br>
Blue Star Building - FIN - D4
<br>
1130 Bruxelles<br>
<br>
Numéro d'identification unique : &nbsp;0418.547.872<br>
Date de l'inscription à la liste : &nbsp;21-09-2010<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="http://www.nbb.be/DOC/CP/ENG/psd/Atos_Worldline_SA.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
<td>3 4 5 7 8</td>
<td></td>
</tr>
<tr>
<td><strong>WorldRemit Belgium</strong><br>
<em>Société anonyme</em><br>
Place Marcel Broodthaers 8 <br>
1060 Bruxelles<br>
<br>
Numéro d'identification unique : &nbsp;0718.634.495<br>
Date de l'inscription à la liste : &nbsp;26-03-2019<br>
Succursales/Agents/Libre prestation des services : &nbsp;<a href="file:///L:/PRXNWEBP/DOC/CP/eng/psd/WorldRemit.docx?t=19411" target="_blank">voyez la liste en annexe</a></td>
<td>6</td>
<td></td>
</tr>
</tbody>
</table>
</div>
</li>
</ul>
<h2 id="bm_Changes12M">
Modifications de la liste au cours des douze derniers mois
</h2>
<div class="changes-12">
<ul>
<li>
<h3>
Etablissements de paiement de droit belge
</h3>
<ul>
<li>
<span class="ChangeTypeTitle">Agrément</span>
<div class="table-responsive">
<table class="table" width="100%" cellpadding="5">
<tbody>
<tr>
<th class="ColumnTitleModification" width="10%" style="word-wrap:normal;word-break:keep-all">Date effective</th>
<th class="ColumnTitleModification" width="90%" style="word-wrap:normal;word-break:keep-all"> </th>
</tr>
<tr>
<td style="white-space:nowrap">19-02-2019</td>
<td style="word-wrap:normal;word-break:keep-all">Isabel </td>
</tr>
<tr>
<td style="white-space:nowrap">19-02-2019</td>
<td style="word-wrap:normal;word-break:keep-all">Let's Didid </td>
</tr>
<tr>
<td style="white-space:nowrap">19-03-2019</td>
<td style="word-wrap:normal;word-break:keep-all">Transferwise Europe </td>
</tr>
<tr>
<td style="white-space:nowrap">26-03-2019</td>
<td style="word-wrap:normal;word-break:keep-all">Money International </td>
</tr>
<tr>
<td style="white-space:nowrap">26-03-2019</td>
<td style="word-wrap:normal;word-break:keep-all">WorldRemit Belgium </td>
</tr>
</tbody>
</table>
</div>
</li>
<li>
<span class="ChangeTypeTitle">Radiation</span>
<div class="table-responsive">
<table class="table" width="100%" cellpadding="5">
<tbody>
<tr>
<th class="ColumnTitleModification" width="10%" style="word-wrap:normal;word-break:keep-all">Date effective</th>
<th class="ColumnTitleModification" width="90%" style="word-wrap:normal;word-break:keep-all"> </th>
</tr>
<tr>
<td style="white-space:nowrap">26-06-2018</td>
<td style="word-wrap:normal;word-break:keep-all">B+S Payment Europe </td>
</tr>
</tbody>
</table>
</div>
</li>
<li>
<span class="ChangeTypeTitle">Changement de la dénomination sociale</span>
<div class="table-responsive">
<table class="table" width="100%" cellpadding="5">
<tbody>
<tr>
<th class="ColumnTitleModification" width="10%" style="word-wrap:normal;word-break:keep-all">Date effective</th>
<th class="ColumnTitleModification" width="30%" style="word-wrap:normal;word-break:keep-all">Ancienne dénomination</th>
<th class="ColumnTitleModification" width="60%" style="word-wrap:normal;word-break:keep-all">Nouvelle dénomination</th>
</tr>
<tr>
<td style="white-space:nowrap">04-01-2019</td>
<td style="word-wrap:normal;word-break:keep-all">Teal IT </td>
<td style="word-wrap:normal;word-break:keep-all">Digiteal </td>
</tr>
<tr>
<td style="white-space:nowrap">01-02-2019</td>
<td style="word-wrap:normal;word-break:keep-all">Bank Card Company Corporate </td>
<td style="word-wrap:normal;word-break:keep-all">Airplus International </td>
</tr>
</tbody>
</table>
</div>
</li>
</ul>
</li>
</ul>
</div>
<p><em>Fin de liste</em></p>
<a href="#PrudentialList">Haut de liste</a>

1324
tests/data/nl/din329_d2_01.html Normal file
View File

File diff suppressed because one or more lines are too long

42
tests/data/nl/din329_d2_01.json Normal file
View File

@ -0,0 +1,42 @@
[
[
"Statutory name",
"NIBC Bank N.V."
],
[
"Trade name",
"NIBC Direct"
],
[
"Address",
"Carnegieplein 4"
],
[
"Postal code",
"2517 KJ"
],
[
"Place of residence",
"'S-GRAVENHAGE"
],
[
"Country",
"Netherlands"
],
[
"Disclosure",
""
],
[
"Chamber of Commerce",
"27032036"
],
[
"Relation number DNB",
"B0685"
],
[
"Category",
"Emittent effecten CSDB, Bank"
]
]

21
tests/data/no/cb_001_ps.html Normal file
View File

@ -0,0 +1,21 @@
<h3 class="license-unit-label">Cross-border services/classes</h3>
<p>
The entity may provide cross-border services/classes from Slovakia
</p>
<div>
<h4 class="license-gov-subheading">To Norway without physical establishment, for the following services/classes</h4>
<ul>
<li>
Payment service activity
<ul>
<li>
Service 3 and 4 in annex to PSD 1
</li>
</ul>
</li>
</ul>
</div>

16
tests/data/no/cb_001_ps.json Normal file
View File

@ -0,0 +1,16 @@
{
"name": "Cross-border services/classes The entity may provide cross-border services/classes from Slovakia",
"data": [
{
"name": "To Norway without physical establishment, for the following services/classes",
"data": [
{
"name": "Payment service activity",
"data": [
"Service 3 and 4 in annex to PSD 1"
]
}
]
}
]
}

36
tests/data/no/cb_002_ps.html Normal file
View File

@ -0,0 +1,36 @@
<h3 class="license-unit-label">Cross-border services/classes</h3>
<p>
The entity may provide cross-border services/classes from United Kingdom
</p>
<div>
<h4 class="license-gov-subheading">To Norway without physical establishment, for the following services/classes</h4>
<ul>
<li>
Payment service activity
<ul>
<li>
Service 6 in annex to PSD 1
</li>
</ul>
</li>
</ul>
</div>
<div>
<h4 class="license-gov-subheading">To Norway through tied agents</h4>
<ul>
<li class="license-link-item">
<a href="?id=192475">HASHYL FINANCIAL SERVICES YAHYA SONKO</a> <span>(Norway)</span>
</li>
<li class="license-link-item">
<a href="?id=192481">SHARANHA GULLSMED Varathalingam Selliah</a> <span>(Norway)</span>
</li>
</ul>
</div>

23
tests/data/no/cb_002_ps.json Normal file
View File

@ -0,0 +1,23 @@
{
"name": "Cross-border services/classes The entity may provide cross-border services/classes from United Kingdom",
"data": [
{
"name": "To Norway without physical establishment, for the following services/classes",
"data": [
{
"name": "Payment service activity",
"data": [
"Service 6 in annex to PSD 1"
]
}
]
},
{
"name": "To Norway through tied agents",
"data": [
"HASHYL FINANCIAL SERVICES YAHYA SONKO (Norway)",
"SHARANHA GULLSMED Varathalingam Selliah (Norway)"
]
}
]
}

24
tests/data/no/cb_003_ps.html Normal file
View File

@ -0,0 +1,24 @@
<h3 class="license-unit-label">Cross-border services/classes</h3>
<p>
The entity may provide cross-border services/classes from United Kingdom
</p>
<div>
<h4 class="license-gov-subheading">To Norway without physical establishment, for the following services/classes</h4>
<ul>
<li>
Payment service activity
<ul>
<li>
Service 3 and 4 in annex to PSD 1
</li>
<li>
Service 5 in annex to PSD 1
</li>
</ul>
</li>
</ul>
</div>

17
tests/data/no/cb_003_ps.json Normal file
View File

@ -0,0 +1,17 @@
{
"name": "Cross-border services/classes The entity may provide cross-border services/classes from United Kingdom",
"data": [
{
"name": "To Norway without physical establishment, for the following services/classes",
"data": [
{
"name": "Payment service activity",
"data": [
"Service 3 and 4 in annex to PSD 1",
"Service 5 in annex to PSD 1"
]
}
]
}
]
}

33
tests/data/no/cb_004_ps.html Normal file
View File

@ -0,0 +1,33 @@
<h3 class="license-unit-label">Cross-border services/classes</h3>
<p>
The entity may provide cross-border services/classes from United Kingdom
</p>
<div>
<h4 class="license-gov-subheading">To Norway without physical establishment, for the following services/classes</h4>
<ul>
<li>
Payment service activity
<ul>
<li>
Service 1 and 2 in annex to PSD 1
</li>
<li>
Service 3 and 4 in annex to PSD 1
</li>
<li>
Service 5 in annex to PSD 1
</li>
<li>
Service 6 in annex to PSD 1
</li>
<li>
Service 7 in annex to PSD 1
</li>
</ul>
</li>
</ul>
</div>

20
tests/data/no/cb_004_ps.json Normal file
View File

@ -0,0 +1,20 @@
{
"name": "Cross-border services/classes The entity may provide cross-border services/classes from United Kingdom",
"data": [
{
"name": "To Norway without physical establishment, for the following services/classes",
"data": [
{
"name": "Payment service activity",
"data": [
"Service 1 and 2 in annex to PSD 1",
"Service 3 and 4 in annex to PSD 1",
"Service 5 in annex to PSD 1",
"Service 6 in annex to PSD 1",
"Service 7 in annex to PSD 1"
]
}
]
}
]
}

32
tests/data/no/cb_005_ps_with_empty_item.html Normal file
View File

@ -0,0 +1,32 @@
<h3 class="license-unit-label">Cross-border services/classes</h3>
<p>
The entity may provide cross-border services/classes from Germany
</p>
<div>
<h4 class="license-gov-subheading">To Norway without physical establishment, for the following services/classes</h4>
<ul>
<li>
<ul>
<li>
Distribution/Redemption of electronic money
</li>
<li>
Issuing electronic money
</li>
</ul>
</li>
<li>
Payment service activity
<ul>
<li>
Service 5 in annex to PSD 1
</li>
</ul>
</li>
</ul>
</div>

23
tests/data/no/cb_005_ps_with_empty_item.json Normal file
View File

@ -0,0 +1,23 @@
{
"name": "Cross-border services/classes The entity may provide cross-border services/classes from Germany",
"data": [
{
"name": "To Norway without physical establishment, for the following services/classes",
"data": [
{
"name": "",
"data": [
"Distribution/Redemption of electronic money",
"Issuing electronic money"
]
},
{
"name": "Payment service activity",
"data": [
"Service 5 in annex to PSD 1"
]
}
]
}
]
}

71
tests/data/no/cb_006_em_with_multi_countries.html Normal file
View File

@ -0,0 +1,71 @@
<h3 class="license-unit-label">Cross-border services/classes</h3>
<p>
The entity may provide cross-border services/classes from Norway
</p>
<div>
<h4 class="license-gov-subheading">To multiple countries without physical establishment, for the following services/classes</h4>
<ul>
<li>
<ul>
<li>
Distribution/Redemption of electronic money
<ul>
<li>
Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom
</li>
</ul>
</li>
<li>
Issuing electronic money
<ul>
<li>
Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom
</li>
</ul>
</li>
</ul>
</li>
<li>
Payment service activity
<ul>
<li>
Service 1 and 2 in annex to PSD 1
<ul>
<li>
Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom
</li>
</ul>
</li>
<li>
Service 3 and 4 in annex to PSD 1
<ul>
<li>
Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom
</li>
</ul>
</li>
<li>
Service 5 in annex to PSD 1
<ul>
<li>
Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom
</li>
</ul>
</li>
<li>
Service 6 in annex to PSD 1
<ul>
<li>
Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom
</li>
</ul>
</li>
</ul>
</li>
</ul>
</div>

44
tests/data/no/cb_006_em_with_multi_countries.json Normal file
View File

@ -0,0 +1,44 @@
{
"name": "Cross-border services/classes The entity may provide cross-border services/classes from Norway",
"data": [
{
"name": "To multiple countries without physical establishment, for the following services/classes",
"data": [
{
"name": "",
"data": [
{
"name": "Distribution/Redemption of electronic money",
"data": [ "Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom" ]
},
{
"name": "Issuing electronic money",
"data": [ "Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom" ]
}
]
},
{
"name": "Payment service activity",
"data": [
{
"name": "Service 1 and 2 in annex to PSD 1",
"data": [ "Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom" ]
},
{
"name": "Service 3 and 4 in annex to PSD 1",
"data": [ "Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom" ]
},
{
"name": "Service 5 in annex to PSD 1",
"data": [ "Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom" ]
},
{
"name": "Service 6 in annex to PSD 1",
"data": [ "Austria, Belgium, Bulgaria, Croatia, Cyprus, Czech Republic, Denmark, Estonia, Finland, France, Germany, Greece, Hungary, Ireland, Italy, Latvia, Liechtenstein, Lithuania, Luxembourg, Malta, Netherlands, Poland, Portugal, Romania, Slovakia, Slovenia, Spain, Sweden, United Kingdom" ]
}
]
}
]
}
]
}

83
tests/data/no/ent_002_cb.json
View File

@ -1,45 +1,50 @@
{
"crossborderServicesclasses": [
"The entity may provide cross-border services/classes from Spain"
],
"name": "Cross-border services/classes The entity may provide cross-border services/classes from Spain",
"data": [
{
"description": {
"main": "main",
"ancillaryServicesMiFidDirective": "Ancillary services (MiFid directive)",
"investmentServicesAndActivitiesMiFidDirective": "Investment services and activities (MiFid directive)"
},
"main": [
"1. Acceptance of deposits and other repayable funds",
"2. Lending",
"3. Financial leasing",
"6. Guarantees and commitments",
"7. a) Money-market instruments",
"7. b) Foreign exchange",
"7. c) Financial future/forward contracts and options",
"7. d) Foreign exchange- and interest rate instruments",
"7. e) Transferable securities",
"8. Participation in securities issues and the provision of services related to such issues",
"9. Advisory services",
"11. Portfolio management and advice"
],
"ancillaryServicesMiFidDirective": [
"No 1: Safekeeping and administration of financial instruments",
"No 2: Granting credits or loans",
"No 3: Advice on capital structure, strategy et al.; advice and services re. to mergers and purchases",
"No 4: Foreign exchange services connected to the provision of investment services",
"No 5: Provision of investment recommendations, analyses and general recommendations",
"No 6: Services related to underwriting",
"No 7: Services related to underlying of derivatives connected to investment and ancillary services"
],
"investmentServicesAndActivitiesMiFidDirective": [
"No 1: Reception and transmission of orders",
"No 2: Execution of orders",
"No 3: Dealing on own account",
"No 4: Active management of investors' portfolios",
"No 5: Investment advice",
"No 6: Underwriting of fin. instruments or placing of fin. instruments on a firm commitment basis",
"No 7: Placing of financial instruments without a firm commitment basis"
"name": "To Norway without physical establishment, for the following services/classes",
"data": [
{
"name": "",
"data": [
"1. Acceptance of deposits and other repayable funds",
"2. Lending",
"3. Financial leasing",
"6. Guarantees and commitments",
"7. a) Money-market instruments",
"7. b) Foreign exchange",
"7. c) Financial future/forward contracts and options",
"7. d) Foreign exchange- and interest rate instruments",
"7. e) Transferable securities",
"8. Participation in securities issues and the provision of services related to such issues",
"9. Advisory services",
"11. Portfolio management and advice"
]
},
{
"name": "Ancillary services (MiFid directive)",
"data": [
"No 1: Safekeeping and administration of financial instruments",
"No 2: Granting credits or loans",
"No 3: Advice on capital structure, strategy et al.; advice and services re. to mergers and purchases",
"No 4: Foreign exchange services connected to the provision of investment services",
"No 5: Provision of investment recommendations, analyses and general recommendations",
"No 6: Services related to underwriting",
"No 7: Services related to underlying of derivatives connected to investment and ancillary services"
]
},
{
"name": "Investment services and activities (MiFid directive)",
"data": [
"No 1: Reception and transmission of orders",
"No 2: Execution of orders",
"No 3: Dealing on own account",
"No 4: Active management of investors' portfolios",
"No 5: Investment advice",
"No 6: Underwriting of fin. instruments or placing of fin. instruments on a firm commitment basis",
"No 7: Placing of financial instruments without a firm commitment basis"
]
}
]
}
]

83
tests/data/no/ent_002_sniff.json
View File

@ -12,47 +12,52 @@
"1/1/2016"
],
"crossBorder": {
"crossborderServicesclasses": [
"The entity may provide cross-border services/classes from Spain"
],
"name": "Cross-border services/classes The entity may provide cross-border services/classes from Spain",
"data": [
{
"description": {
"main": "main",
"ancillaryServicesMiFidDirective": "Ancillary services (MiFid directive)",
"investmentServicesAndActivitiesMiFidDirective": "Investment services and activities (MiFid directive)"
},
"main": [
"1. Acceptance of deposits and other repayable funds",
"2. Lending",
"3. Financial leasing",
"6. Guarantees and commitments",
"7. a) Money-market instruments",
"7. b) Foreign exchange",
"7. c) Financial future/forward contracts and options",
"7. d) Foreign exchange- and interest rate instruments",
"7. e) Transferable securities",
"8. Participation in securities issues and the provision of services related to such issues",
"9. Advisory services",
"11. Portfolio management and advice"
],
"ancillaryServicesMiFidDirective": [
"No 1: Safekeeping and administration of financial instruments",
"No 2: Granting credits or loans",
"No 3: Advice on capital structure, strategy et al.; advice and services re. to mergers and purchases",
"No 4: Foreign exchange services connected to the provision of investment services",
"No 5: Provision of investment recommendations, analyses and general recommendations",
"No 6: Services related to underwriting",
"No 7: Services related to underlying of derivatives connected to investment and ancillary services"
],
"investmentServicesAndActivitiesMiFidDirective": [
"No 1: Reception and transmission of orders",
"No 2: Execution of orders",
"No 3: Dealing on own account",
"No 4: Active management of investors' portfolios",
"No 5: Investment advice",
"No 6: Underwriting of fin. instruments or placing of fin. instruments on a firm commitment basis",
"No 7: Placing of financial instruments without a firm commitment basis"
"name": "To Norway without physical establishment, for the following services/classes",
"data": [
{
"name": "",
"data": [
"1. Acceptance of deposits and other repayable funds",
"2. Lending",
"3. Financial leasing",
"6. Guarantees and commitments",
"7. a) Money-market instruments",
"7. b) Foreign exchange",
"7. c) Financial future/forward contracts and options",
"7. d) Foreign exchange- and interest rate instruments",
"7. e) Transferable securities",
"8. Participation in securities issues and the provision of services related to such issues",
"9. Advisory services",
"11. Portfolio management and advice"
]
},
{
"name": "Ancillary services (MiFid directive)",
"data": [
"No 1: Safekeeping and administration of financial instruments",
"No 2: Granting credits or loans",
"No 3: Advice on capital structure, strategy et al.; advice and services re. to mergers and purchases",
"No 4: Foreign exchange services connected to the provision of investment services",
"No 5: Provision of investment recommendations, analyses and general recommendations",
"No 6: Services related to underwriting",
"No 7: Services related to underlying of derivatives connected to investment and ancillary services"
]
},
{
"name": "Investment services and activities (MiFid directive)",
"data": [
"No 1: Reception and transmission of orders",
"No 2: Execution of orders",
"No 3: Dealing on own account",
"No 4: Active management of investors' portfolios",
"No 5: Investment advice",
"No 6: Underwriting of fin. instruments or placing of fin. instruments on a firm commitment basis",
"No 7: Placing of financial instruments without a firm commitment basis"
]
}
]
}
]

86
tests/data/no/ent_004_sniff.json
View File

@ -11,49 +11,55 @@
"registrationDate": [
"1/1/2016"
],
"crossBorder": {
"crossborderServicesclasses": [
"The entity may provide cross-border services/classes from Luxembourg"
],
"name": "Cross-border services/classes The entity may provide cross-border services/classes from Luxembourg",
"data": [
{
"description": {
"main": "main",
"ancillaryServicesMiFidDirective": "Ancillary services (MiFid directive)",
"investmentServicesAndActivitiesMiFidDirective": "Investment services and activities (MiFid directive)"
},
"main": [
"1. Acceptance of deposits and other repayable funds",
"2. Lending",
"3. Financial leasing",
"4. Money transmission services",
"5. Issuing and administering of payments (credit cards etc.)",
"6. Guarantees and commitments",
"7. a) Money-market instruments",
"7. b) Foreign exchange",
"7. c) Financial future/forward contracts and options",
"7. d) Foreign exchange- and interest rate instruments",
"7. e) Transferable securities",
"8. Participation in securities issues and the provision of services related to such issues",
"9. Advisory services",
"10. Money broking",
"11. Portfolio management and advice",
"12. Safekeeping and administration of securities",
"13. Credit reference services",
"14. Safe custody services"
],
"ancillaryServicesMiFidDirective": [
"No 1: Safekeeping and administration of financial instruments",
"No 2: Granting credits or loans",
"No 4: Foreign exchange services connected to the provision of investment services",
"No 5: Provision of investment recommendations, analyses and general recommendations",
"No 7: Services related to underlying of derivatives connected to investment and ancillary services"
],
"investmentServicesAndActivitiesMiFidDirective": [
"No 1: Reception and transmission of orders",
"No 2: Execution of orders",
"No 4: Active management of investors' portfolios",
"No 5: Investment advice"
"name": "To Norway without physical establishment, for the following services/classes",
"data": [
{
"name": "",
"data": [
"1. Acceptance of deposits and other repayable funds",
"2. Lending",
"3. Financial leasing",
"4. Money transmission services",
"5. Issuing and administering of payments (credit cards etc.)",
"6. Guarantees and commitments",
"7. a) Money-market instruments",
"7. b) Foreign exchange",
"7. c) Financial future/forward contracts and options",
"7. d) Foreign exchange- and interest rate instruments",
"7. e) Transferable securities",
"8. Participation in securities issues and the provision of services related to such issues",
"9. Advisory services",
"10. Money broking",
"11. Portfolio management and advice",
"12. Safekeeping and administration of securities",
"13. Credit reference services",
"14. Safe custody services"
]
},
{
"name": "Ancillary services (MiFid directive)",
"data": [
"No 1: Safekeeping and administration of financial instruments",
"No 2: Granting credits or loans",
"No 4: Foreign exchange services connected to the provision of investment services",
"No 5: Provision of investment recommendations, analyses and general recommendations",
"No 7: Services related to underlying of derivatives connected to investment and ancillary services"
]
},
{
"name": "Investment services and activities (MiFid directive)",
"data": [
"No 1: Reception and transmission of orders",
"No 2: Execution of orders",
"No 4: Active management of investors' portfolios",
"No 5: Investment advice"
]
}
]
}
]

10
tests/data/no/ent_005_cb.json
View File

@ -1,13 +1,9 @@
{
"crossborderServicesclasses": [
"The entity may provide cross-border services/classes from Sweden"
],
"name": "Cross-border services/classes The entity may provide cross-border services/classes from Sweden",
"data": [
{
"description": {
"toNorwayThroughABranchEstablishment": "To Norway through a branch establishment"
},
"toNorwayThroughABranchEstablishment": [
"name": "To Norway through a branch establishment",
"data": [
"AVIDA FINANS AB NUF (Norway)"
]
}

11
tests/data/no/ent_005_sniff.json
View File

@ -12,19 +12,16 @@
"8/4/2008"
],
"crossBorder": {
"crossborderServicesclasses": [
"The entity may provide cross-border services/classes from Sweden"
],
"name": "Cross-border services/classes The entity may provide cross-border services/classes from Sweden",
"data": [
{
"description": {
"toNorwayThroughABranchEstablishment": "To Norway through a branch establishment"
},
"toNorwayThroughABranchEstablishment": [
"name": "To Norway through a branch establishment",
"data": [
"AVIDA FINANS AB NUF (Norway)"
]
}
]
}
}
]

23
tests/data/no/ent_006_cb.json
View File

@ -1,21 +1,20 @@
{
"crossborderServicesclasses": [
"The entity may provide cross-border services/classes from United Kingdom"
],
"name": "Cross-border services/classes The entity may provide cross-border services/classes from United Kingdom",
"data": [
{
"description": {
"paymentServiceActivity": "Payment service activity"
},
"paymentServiceActivity": [
"Service 6 in annex to PSD 1"
"name": "To Norway without physical establishment, for the following services/classes",
"data": [
{
"name": "Payment service activity",
"data": [
"Service 6 in annex to PSD 1"
]
}
]
},
{
"description": {
"toNorwayThroughTiedAgents": "To Norway through tied agents"
},
"toNorwayThroughTiedAgents": [
"name": "To Norway through tied agents",
"data": [
"HASHYL FINANCIAL SERVICES YAHYA SONKO (Norway)",
"SHARANHA GULLSMED Varathalingam Selliah (Norway)"
]

23
tests/data/no/ent_006_sniff.json
View File

@ -12,23 +12,22 @@
"1/1/2016"
],
"crossBorder": {
"crossborderServicesclasses": [
"The entity may provide cross-border services/classes from United Kingdom"
],
"name": "Cross-border services/classes The entity may provide cross-border services/classes from United Kingdom",
"data": [
{
"description": {
"paymentServiceActivity": "Payment service activity"
},
"paymentServiceActivity": [
"Service 6 in annex to PSD 1"
"name": "To Norway without physical establishment, for the following services/classes",
"data": [
{
"name": "Payment service activity",
"data": [
"Service 6 in annex to PSD 1"
]
}
]
},
{
"description": {
"toNorwayThroughTiedAgents": "To Norway through tied agents"
},
"toNorwayThroughTiedAgents": [
"name": "To Norway through tied agents",
"data": [
"HASHYL FINANCIAL SERVICES YAHYA SONKO (Norway)",
"SHARANHA GULLSMED Varathalingam Selliah (Norway)"
]

52
tests/scrape.at.js Normal file
View File

@ -0,0 +1,52 @@
const tape = require('tape');
const _test = require('tape-promise').default; // <---- notice 'default'
const test = _test(tape); // decorate tape
const diff = require('deep-diff');
const fs = require('fs');
const jsonfile = require('jsonfile');
const Austria = require('../ncas/at');
const atScraper = new Austria();
test.test('Austria 🇦🇹 :: Tests', async t => {
t.test('Test extractEntityDetails ent_001', async t => {
const html = fs.readFileSync('tests/data/at/ent_001.html');
const output = await atScraper.extractEntityDetails(html);
const expectedJSON = jsonfile.readFileSync('tests/data/at/ent_001.json');
t.deepEquals(output, expectedJSON, 'Extracted entity details from Page');
console.log(diff(output, expectedJSON));
t.end();
});
t.test('Test extractEntityDetails ent_002', async t => {
const html = fs.readFileSync('tests/data/at/ent_002.html');
const output = await atScraper.extractEntityDetails(html);
const expectedJSON = jsonfile.readFileSync('tests/data/at/ent_002.json');
t.deepEquals(output, expectedJSON, 'Extracted entity details from Page');
console.log(diff(output, expectedJSON));
t.end();
});
t.test('Test extractEntityDetails ent_003', async t => {
const html = fs.readFileSync('tests/data/at/ent_003.html');
const output = await atScraper.extractEntityDetails(html);
const expectedJSON = jsonfile.readFileSync('tests/data/at/ent_003.json');
t.deepEquals(output, expectedJSON, 'Extracted entity details from Page');
console.log(diff(output, expectedJSON));
t.end();
});
t.end();
});

166
tests/scrape.be.js Normal file
View File

@ -0,0 +1,166 @@
const cheerio = require('cheerio');
const tape = require('tape');
const _test = require('tape-promise').default; // <---- notice 'default'
const test = _test(tape); // decorate tape
const fs = require('fs');
const jsonfile = require('jsonfile');
const Belgium = require('../ncas/be');
const beScraper = new Belgium();
test.test('Entities', async t => {
t.test('Extract main details...', async t => {
t.test('...from td container', async t => {
const htmlFile = 'tests/data/be/ps_fullpage_001.html';
const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' });
const $ = cheerio.load(html, { 'decodeEntities': false });
const detailsContainer = $('ul.List1 div.table-responsive tbody tr td').eq(0);
const output = await beScraper.extractMainDetails(detailsContainer);
const expectedJSON = jsonfile.readFileSync('tests/data/be/ps_001_mainDetails.json');
t.deepEquals(output, expectedJSON);
t.end();
});
t.test('...from li container', async t => {
const htmlFile = 'tests/data/be/ci_fullpage_001.html';
const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' });
const $ = cheerio.load(html, { 'decodeEntities': false });
const detailsContainer = $('ul.List1 ul.List2 > li > ul > li').eq(0);
const output = await beScraper.extractMainDetails(detailsContainer);
const expectedJSON = jsonfile.readFileSync('tests/data/be/ci_001_mainDetails.json');
t.deepEquals(output, expectedJSON);
t.end();
});
t.test('...from unusual entity (3-line address and large spacing)', async t => {
const htmlFile = 'tests/data/be/em_fullpage_001.html';
const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' });
const $ = cheerio.load(html, { 'decodeEntities': false });
const detailsContainer = $('ul.List1 div.table-responsive tbody tr').eq(4).children('td').eq(0);
const output = await beScraper.extractMainDetails(detailsContainer);
const expectedJSON = jsonfile.readFileSync('tests/data/be/em_001_mainDetails.json');
t.deepEquals(output, expectedJSON);
t.end();
});
t.end();
});
t.test('Extract full details...', async t => {
t.test('...from payment service', async t => {
const htmlFile = 'tests/data/be/ps_fullpage_001.html';
const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' });
const $ = cheerio.load(html, { 'decodeEntities': false });
const fullDetailsContainer = $('ul.List1 div.table-responsive tbody tr').eq(0);
const output = await beScraper.extractFullDetails(fullDetailsContainer, 0);
const expectedJSON = jsonfile.readFileSync('tests/data/be/ps_001_fullDetails.json');
t.deepEquals(output, expectedJSON);
t.end();
});
t.test('...from emoney service', async t => {
const htmlFile = 'tests/data/be/em_fullpage_001.html';
const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' });
const $ = cheerio.load(html, { 'decodeEntities': false });
const fullDetailsContainer = $('ul.List1 div.table-responsive tbody tr').eq(0);
const output = await beScraper.extractFullDetails(fullDetailsContainer, 0);
const expectedJSON = jsonfile.readFileSync('tests/data/be/em_001_fullDetails.json');
t.deepEquals(output, expectedJSON);
t.end();
});
t.test('...from credit institution', async t => {
const htmlFile = 'tests/data/be/ci_fullpage_001.html';
const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' });
const $ = cheerio.load(html, { 'decodeEntities': false });
const fullDetailsContainer = $('ul.List1 ul.List2 > li > ul > li').eq(0);
const output = await beScraper.extractFullDetails(fullDetailsContainer, 2);
const expectedJSON = jsonfile.readFileSync('tests/data/be/ci_001_fullDetails.json');
t.deepEquals(output, expectedJSON);
t.end();
});
t.end();
});
t.test('Extract entities from container...', async t => {
t.test('...of payment services (tbody)', async t => {
const htmlFile = 'tests/data/be/ps_fullpage_001.html';
const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' });
const $ = cheerio.load(html, { 'decodeEntities': false });
const entitiesContainer = $('ul.List1 tbody');
const output = await beScraper.extractEntitiesFromContainer(entitiesContainer, 0);
t.equals(output.length, 24);
t.end();
});
t.test('...of credit institutions (ul)', async t => {
const htmlFile = 'tests/data/be/ci_fullpage_001.html';
const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' });
const $ = cheerio.load(html, { 'decodeEntities': false });
const entitiesContainer = $('ul.List1 ul.List2 > li > ul').eq(0); // get the first list only for this test
const output = await beScraper.extractEntitiesFromContainer(entitiesContainer, 2);
t.equals(output.length, 25);
t.end();
});
t.end();
});
t.test('Extract index...', async t => {
t.test('...of payment services', async t => {
const htmlFile = 'tests/data/be/ps_fullpage_001.html';
const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' });
const $ = cheerio.load(html, { 'decodeEntities': false });
const indexContainer = $('#PrudentialList');
const output = await beScraper.extractIndex(indexContainer, 0);
// console.log(output);
t.end();
});
t.test('...of credit institutions', async t => {
const htmlFile = 'tests/data/be/ci_fullpage_001.html';
const html = fs.readFileSync(htmlFile, { 'encoding': 'utf-8' });
const $ = cheerio.load(html, { 'decodeEntities': false });
const indexContainer = $('#PrudentialList');
const output = await beScraper.extractIndex(indexContainer, 2);
// console.log(output);
t.end();
});
t.end();
});
t.end();
});

67
tests/scrape.fr.js
View File

@ -98,7 +98,7 @@ test('FRANCE:: Scrape Indexes', async t => {
t.end();
});
t.end();
t.end();
});
test('FRANCE Scrape a Payment Instititute', async t => {
@ -222,61 +222,54 @@ test('FRANCE Breaking CI 001', async t => {
});
test('FRANCE:: DIR-3741', async t => {
const dir3741_001 = fs.readFileSync('tests/data/fr/dir3741_001.html');
const dir3741_001Data = jsonfile.readFileSync('tests/data/fr/dir3741_001.json');
const dir3741001 = fs.readFileSync('tests/data/fr/dir3741_001.html');
const dir3741001Data = jsonfile.readFileSync('tests/data/fr/dir3741_001.json');
const dir3741_002 = fs.readFileSync('tests/data/fr/dir3741_002.html');
const dir3741_002Data = jsonfile.readFileSync('tests/data/fr/dir3741_002.json');
const dir3741002 = fs.readFileSync('tests/data/fr/dir3741_002.html');
const dir3741002Data = jsonfile.readFileSync('tests/data/fr/dir3741_002.json');
const dir3741_003 = fs.readFileSync('tests/data/fr/dir3741_003.html');
const dir3741_003Data = jsonfile.readFileSync('tests/data/fr/dir3741_003.json');
const dir3741003 = fs.readFileSync('tests/data/fr/dir3741_003.html');
const dir3741003Data = jsonfile.readFileSync('tests/data/fr/dir3741_003.json');
const frScraper = new France();
t.test('FRANCE::Extract Details from Page 1/103', async t => {
const $ = cheerio.load(dir3741_001);
const $table = $('table.table tr');
const $ = cheerio.load(dir3741001);
const $table = $('table.table tr');
const links = await frScraper.extractLinks($table, true);
const links = await frScraper.extractLinks($table, true);
const linkCount = links.length;
const linkCount = links.length;
t.equal(linkCount, 1, 'Scrapes the correct number of links (1)');
t.deepEquals(links, dir3741_001Data, 'Links match the data');
t.end();
t.equal(linkCount, 1, 'Scrapes the correct number of links (1)');
t.deepEquals(links, dir3741001Data, 'Links match the data');
t.end();
});
t.test('FRANCE::Extract Details from Page 4/103', async t => {
const $ = cheerio.load(dir3741_002);
const $table = $('table.table tr');
const $ = cheerio.load(dir3741002);
const $table = $('table.table tr');
const links = await frScraper.extractLinks($table, true);
const links = await frScraper.extractLinks($table, true);
const linkCount = links.length;
const linkCount = links.length;
t.equal(linkCount, 3, 'Scrapes the correct number of links (3)');
t.deepEquals(links, dir3741_002Data, 'Links match the data');
t.end();
});
t.equal(linkCount, 3, 'Scrapes the correct number of links (3)');
t.deepEquals(links, dir3741002Data, 'Links match the data');
t.end();
});
t.test('FRANCE::Extract Details from Page 11/103', async t => {
const $ = cheerio.load(dir3741_003);
const $table = $('table.table tr');
const $ = cheerio.load(dir3741003);
const $table = $('table.table tr');
const links = await frScraper.extractLinks($table, true);
const links = await frScraper.extractLinks($table, true);
const linkCount = links.length;
const linkCount = links.length;
t.equal(linkCount, 2, 'Scrapes the correct number of links (2)');
t.deepEquals(links, dir3741_003Data, 'Links match the data');
t.end();
});
t.equal(linkCount, 2, 'Scrapes the correct number of links (2)');
t.deepEquals(links, dir3741003Data, 'Links match the data');
t.end();
});
t.end();
});

16
tests/scrape.nl.js
View File

@ -315,3 +315,19 @@ test('NL:: Scrape a Credit Service', async t => {
t.end();
});
test('NL:: DIN-329 defects', async t => {
const defect2AHTML = fs.readFileSync('tests/data/nl/din329_d2_01.html');
t.test('NL::Defect 2', async t => {
const expectedJSON = jsonfile.readFileSync('tests/data/nl/din329_d2_01.json');
const output = await nlScraper.extractDetail(defect2AHTML);
t.deepEquals(output, expectedJSON, 'Extracted Details from Page correctly');
t.end();
});
t.end();
});

71
tests/scrape.no.js
View File

@ -225,6 +225,77 @@ test.test('Entity', async t => {
t.end();
});
test.test('NO:: Cross-Border format', async t => {
t.test('NO:: Extract Cross-Border services 001 (PS)', async t => {
const crossborderHtml = fs.readFileSync('tests/data/no/cb_001_ps.html').toString();
const expectedJSON = jsonfile.readFileSync('tests/data/no/cb_001_ps.json');
const output = await noScraper.recurseCrossborderHtml(crossborderHtml);
t.deepEquals(output, expectedJSON, 'Extracted cross-border services from html');
t.end();
});
t.test('NO:: Extract Cross-Border services 002 (PS)', async t => {
const crossborderHtml = fs.readFileSync('tests/data/no/cb_002_ps.html').toString();
const expectedJSON = jsonfile.readFileSync('tests/data/no/cb_002_ps.json');
const output = await noScraper.recurseCrossborderHtml(crossborderHtml);
t.deepEquals(output, expectedJSON, 'Extracted cross-border services from html');
t.end();
});
t.test('NO:: Extract Cross-Border services 003 (PS)', async t => {
const crossborderHtml = fs.readFileSync('tests/data/no/cb_003_ps.html').toString();
const expectedJSON = jsonfile.readFileSync('tests/data/no/cb_003_ps.json');
const output = await noScraper.recurseCrossborderHtml(crossborderHtml);
t.deepEquals(output, expectedJSON, 'Extracted cross-border services from html');
t.end();
});
t.test('NO:: Extract Cross-Border services 004 (PS)', async t => {
const crossborderHtml = fs.readFileSync('tests/data/no/cb_004_ps.html').toString();
const expectedJSON = jsonfile.readFileSync('tests/data/no/cb_004_ps.json');
const output = await noScraper.recurseCrossborderHtml(crossborderHtml);
t.deepEquals(output, expectedJSON, 'Extracted cross-border services from html');
t.end();
});
t.test('NO:: Extract Cross-Border services 005 (PS with an empty list item)', async t => {
const crossborderHtml = fs.readFileSync('tests/data/no/cb_005_ps_with_empty_item.html').toString();
const expectedJSON = jsonfile.readFileSync('tests/data/no/cb_005_ps_with_empty_item.json');
const output = await noScraper.recurseCrossborderHtml(crossborderHtml);
t.deepEquals(output, expectedJSON, 'Extracted cross-border services from html');
t.end();
});
t.test('NO:: Extract Cross-Border services 006 (PS with multiple countries)', async t => {
const crossborderHtml = fs.readFileSync('tests/data/no/cb_006_em_with_multi_countries.html').toString();
const expectedJSON = jsonfile.readFileSync('tests/data/no/cb_006_em_with_multi_countries.json');
const output = await noScraper.recurseCrossborderHtml(crossborderHtml);
t.deepEquals(output, expectedJSON, 'Extracted cross-border services from html');
t.end();
});
t.end();
});
t.end();
});

2
version.properties
View File

@ -1 +1 @@
VERSION=1.0.1
VERSION=1.0.2