replaced request with got

This commit is contained in:
Martin Donnelly 2020-06-01 09:56:48 +01:00
parent 229e2d37b9
commit f7303c3995

View File

@ -9,6 +9,8 @@
const cheerio = require('cheerio');
const request = require('request');
const got = require('got');
const axios = require('axios');
const fecha = require('fecha');
@ -33,7 +35,6 @@ class MasterScraper {
'proxy' : 'http://uk.proxymesh.com:31280',
'tunnel' : true
};
}
setStartUrl(newUrl) {
@ -57,6 +58,14 @@ class MasterScraper {
console.log(options);
got(options.url).then((response) => {
resolve(response.body);
})
.catch((e) => {
reject(e.response.body);
});
/*
request(options, (err, _res, body) => {
if (!err)
resolve(body);
@ -64,6 +73,7 @@ class MasterScraper {
reject(err);
});
*/
});
};
@ -109,7 +119,6 @@ class MasterScraper {
console.error(`${this.siteid} db error`);
console.error(err.message || 'Some error occurred while querying the database.');
});
}
console.log(`- ${this.siteid} addToDB`);
}