const AWS = require('aws-sdk'); const { parse, generate } = require('csv'); class CsvData { constructor() { this.s3 = new AWS.S3(); // defaults for bucket and main CSV name this.bucketName = process.env.OB_SCRAPE_BUCKET || 'obregstoretest'; this.keyName = process.env.OB_SCRAPE_KEYNAME || 'artefacts/UK/FCA/latest/Firms with PSD Permissions (CSV).csv'; } async _getCsvDataFromS3() { return this.s3.getObject({ 'Bucket': this.bucketName, 'Key': this.keyName }).promise(); } async _parseCsvBufferData(csvBufferData) { return new Promise((resolve, reject) => { parse( csvBufferData.Body.toString('utf-8'), { 'columns': true, // required to create objects (instead of array of strings), auto-discovery works fine 'relax_column_count': true // needs to set to false as header have info when file was generated as extra column }, (err, data) => { if (err) return reject(err); resolve(data); } ); }); } async _getFrnAndName(data) { return data.map(el => ({ 'frn': el.FRN, 'firm': el.Firm })); } async getOrgIds() { const rawCsv = await this._getCsvDataFromS3(); const parsedCsv = await this._parseCsvBufferData(rawCsv); const frnIds = this._getFrnAndName(parsedCsv); return frnIds; } } module.exports = CsvData;