Rinser/fixer.js

const querystring = require('querystring');
const url = require('url');

let list = [

  {
    "count": 100,
    "feed_url": "http://www.jobserve.com/MySearch/BAEBF3BDF82B8FEF.rss",
    "page": "https://www.jobserve.com/gb/en/JobSearch.aspx?shid=A7ACEE7915E274717C"
  },
  {

    "count": 100,
    "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=47820652"
  },

  {

    "count": 100,
    "feed_url": "http://www.jobserve.com/MySearch/6DA9769BA89834AA.rss"
  },
  {

    "count": 100,
    "feed_url": "http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss"
  },
  {

    "count": 100,
    "feed_url": "http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss"
  },
  {

    "count": 100,
    "feed_url": "http://www.JobServe.com/MySearch/C7B25D86D0844A.rss"
  },
  {

    "count": 100,
    "feed_url": "http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss"
  },
  {

    "count": 100,
    "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564698"
  },
  {

    "count": 100,
    "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564712"
  },
  {

    "count": 100,
    "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21942123"
  },
  {

    "count": 100,
    "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33166238"
  },
  {

    "count": 100,
    "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888173"
  },
  {

    "count": 100,
    "feed_url": "http://www.s1jobs.com/xml/m7dp711z2r.xml"
  },
  {

    "count": 100,
    "feed_url": "http://www.s1jobs.com/xml/pfvf7o7z2r.xml"
  },
  {

    "count": 100,
    "feed_url": "http://www.s1jobs.com/xml/lluqnt8z2r.xml"
  },
  {

    "count": 100,
    "feed_url": "http://www.s1jobs.com/xml/tu33qt8z2r.xml"
  },
  {

    "count": 100,
    "feed_url": "http://www.s1jobs.com/xml/u3btnz8z2r.xml"
  },
  {

    "count": 100,
    "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33256062"
  },
  {

    "count": 100,
    "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33450169"
  },
  {

    "count": 100,
    "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34517029"
  },
  {

    "count": 100,
    "feed_url": "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888105"
  },
  {

    "count": 100,
    "feed_url": "http://www.technojobs.co.uk/rss.php/glasgow/searchtypeand/locationScotland/sortbyrelevant/jobtypeall"
  },
  {

    "count": 100,
    "feed_url": "http://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss"
  },
  {

    "count": 100,
    "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=London&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CO_LONDON&compare_search=London&search_emp_mkt_cd=ALL"
  },
  {

    "count": 100,
    "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Glasgow&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=TO_G1_GLASGOW&compare_search=Glasgow&search_emp_mkt_cd=ALL"
  },
  {

    "count": 100,
    "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&search_emp_mkt_cd=ALL"
  },
  {

    "count": 100,
    "feed_url": "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Germany&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CY_GERMANY&compare_search=Germany&search_emp_mkt_cd=ALL"
  },
  {

    "count": 100,
    "feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml"
  },
  {

    "count": 100,
    "feed_url": "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml"
  },
  {

    "count": 100,
    "feed_url": "http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss"
  },
  {

    "count": 100,
    "feed_url": "http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml"
  },
  {

    "count": 100,
    "feed_url": "http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml"
  }

  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.JobServe.com/MySearch/EDF47BEA6B31EF.rss"
  },
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.JobServe.com/MySearch/3CAD044BEF2BFA.rss"
  },
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.JobServe.com/MySearch/C7B25D86D0844A.rss"
  },
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.JobServe.com/MySearch/64A3EEF615FA4C.rss"
  },
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564698"
  }
  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21564712"
  }
  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=21942123"
  }
  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33166238"
  }
  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888173"
  }
  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.s1jobs.com/xml/m7dp711z2r.xml"
  }
  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.s1jobs.com/xml/pfvf7o7z2r.xml"
  }
  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.s1jobs.com/xml/lluqnt8z2r.xml"
  }
  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.s1jobs.com/xml/tu33qt8z2r.xml"
  }
  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.s1jobs.com/xml/u3btnz8z2r.xml"
  },
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33256062"
  },
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=33450169"
  }
  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34517029"
  },
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.purelyit.co.uk/cgi-bin/advsearch?rss_feed=1&daysback=1&jbe_id=34888105"
  }
  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.technojobs.co.uk/rss.php/glasgow/searchtypeand/locationScotland/sortbyrelevant/jobtypeall"
  }
  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.jobserve.com/MySearch/6FC7E9ED5F042ECB.rss"
  }
  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html,%20asp,%20web,%20sql,%20delphi,%20vb,%20vbscript,%20php,%20ajax,%20mysql,%20sqlserver,%20javascript,%20intranet,%20vmware,%20virtulization&location_include=Abu%20Dhabi&compare_resolved=RE_ABUDHABI_UNITEDARABEMIRATES&compare_search=Abu%20Dhabi&jobtype=X&search_emp_mkt_cd=ALL"
  }
  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=London&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CO_LONDON&compare_search=London&search_emp_mkt_cd=ALL"
  }
  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Glasgow&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=TO_G1_GLASGOW&compare_search=Glasgow&search_emp_mkt_cd=ALL"
  }
  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&search_emp_mkt_cd=ALL"
  } ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.jobsite.co.uk/cgi-bin/advsearch?rss_feed=1&skill_atleast=html%20,%20asp%20,%20web%20,%20sql%20,%20delphi%20,%20vb%20,%20vbscript%20,%20php%20,%20ajax%20,%20mysql%20,%20sqlserver%20,%20javascript%20,%20intranet%20,%20vmware%20,%20virtulization&location_include=Germany&location_within=10&reqd_salary=ANY|&daysback=7&scc=UK&compare_resolved=CY_GERMANY&compare_search=Germany&search_emp_mkt_cd=ALL"
  }
  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml"
  }
  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.s1jobs.com/xml/b1d7e6c3a9a11964z3r.xml"
  }
  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.jobisjob.co.uk/rss?what=web+developer+javascript&where=glasgow&jobType=Contract"
  }
  ,
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.jobisjob.co.uk/rss?what=javascript&where=glasgow&jobType=Contract"
  },
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.jobisjob.co.uk/rss?what=asp&where=glasgow&jobType=Contract%2Ftemp"
  },
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.jobserve.com/MySearch/CA49421A86CA3F74.rss"
  },
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.s1jobs.com/xml/ddeded091b6f6d33z3r.xml"
  },
  {
    /* "name"              : "JobServe",*/
    "count"             : 100,
    "feed_url"          : "http://www.s1jobs.com/xml/3eafc1ea20f1ca02z3r.xml"
  }

];


let newArray = list.map((item) => {
  let u = url.parse(item.feed_url);
  let newUrl = item.feed_url;
  if (u.host !== '') {
    newUrl = `http://52.211.105.9:6566/${querystring.escape(item.feed_url)}`;
  }
  return newUrl

});

console.log('New array', newArray.length);
let unique = [];

for (let item of newArray) {
  if (unique.indexOf(item) === -1) {
    unique.push(item);
  }
}
console.log('unique', unique.length);

unique = unique.map((item) => {
  return {'count':100, 'feed_url':item};
})
console.log(JSON.stringify(unique));