54 lines
1.4 KiB
JavaScript
54 lines
1.4 KiB
JavaScript
|
const AWS = require('aws-sdk');
|
||
|
|
const { parse, generate } = require('csv');
|
||
|
|
|
||
|
|
class CsvData {
|
||
|
|
|
||
|
|
constructor() {
|
||
|
|
this.s3 = new AWS.S3();
|
||
|
|
|
||
|
|
// defaults for bucket and main CSV name
|
||
|
|
this.bucketName = process.env.OB_SCRAPE_BUCKET || 'obregstoretest';
|
||
|
|
this.keyName = process.env.OB_SCRAPE_KEYNAME || 'artefacts/UK/FCA/latest/Firms with PSD Permissions (CSV).csv';
|
||
|
|
}
|
||
|
|
|
||
|
|
async _getCsvDataFromS3() {
|
||
|
|
return this.s3.getObject({
|
||
|
|
'Bucket': this.bucketName,
|
||
|
|
'Key': this.keyName
|
||
|
|
}).promise();
|
||
|
|
}
|
||
|
|
|
||
|
|
async _parseCsvBufferData(csvBufferData) {
|
||
|
|
return new Promise((resolve, reject) => {
|
||
|
|
parse(
|
||
|
|
csvBufferData.Body.toString('utf-8'),
|
||
|
|
{
|
||
|
|
'columns': true, // required to create objects (instead of array of strings), auto-discovery works fine
|
||
|
|
'relax_column_count': true // needs to set to false as header have info when file was generated as extra column
|
||
|
|
},
|
||
|
|
(err, data) => {
|
||
|
|
if (err) return reject(err);
|
||
|
|
resolve(data);
|
||
|
|
}
|
||
|
|
);
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
async _getFrnAndName(data) {
|
||
|
|
return data.map(el => ({
|
||
|
|
'frn': el.FRN,
|
||
|
|
'firm': el.Firm
|
||
|
|
}));
|
||
|
|
}
|
||
|
|
|
||
|
|
async getOrgIds() {
|
||
|
|
const rawCsv = await this._getCsvDataFromS3();
|
||
|
|
const parsedCsv = await this._parseCsvBufferData(rawCsv);
|
||
|
|
const frnIds = this._getFrnAndName(parsedCsv);
|
||
|
|
|
||
|
|
return frnIds;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
module.exports = CsvData;
|