Computer history and quotes fixed
This commit is contained in:
Martin Donnelly
parent
fb862533dc
commit
0b337a6234
@ -43,12 +43,21 @@ module.exports = {
|
||||
// Throw err;
|
||||
|
||||
const $ = cheerio.load(body);
|
||||
const tdihbody = $('#tdihbody .tdihevent');
|
||||
|
||||
const output = [];
|
||||
|
||||
tdihbody.find('p').each(function(div) {
|
||||
const s = $(this).text();
|
||||
output.push(STRING(s).collapseWhitespace().s);
|
||||
const bigBlock = $('#chm-exhibit-content > article > div > div');
|
||||
|
||||
const rows = $(bigBlock).find('div.chm-exhibit-timeline-decade-row');
|
||||
|
||||
rows.each((i, item) => {
|
||||
const title = $(item).find('.chm-tdih-entry-title').text();
|
||||
const content = $(item).find('.chm-tdih-entry-content').text();
|
||||
|
||||
logger.debug(title);
|
||||
|
||||
output.push(title);
|
||||
output.push(content);
|
||||
});
|
||||
|
||||
return resolve(output);
|
||||
|
||||
@ -7,14 +7,16 @@ const STRING = require('string');
|
||||
const logger = require('log4js').getLogger('quotes');
|
||||
|
||||
const options = {
|
||||
'host': 'andruxnet-random-famous-quotes.p.mashape.com',
|
||||
'path': '/?cat=famous',
|
||||
'host': 'quotes.rest',
|
||||
'path': '/qod.json',
|
||||
|
||||
/* ,
|
||||
'headers': {
|
||||
'accept': 'application/json',
|
||||
'X-Mashape-Key': '5A0H980jK6mshSFL24ZmfiRrNHV2p1d1fhQjsngtx8QWuO9oe4',
|
||||
'Content-Type': 'application/x-www-form-urlencoded'
|
||||
|
||||
},
|
||||
}*/
|
||||
'method': 'GET'
|
||||
};
|
||||
|
||||
@ -31,10 +33,18 @@ module.exports = {
|
||||
response.on('end', function () {
|
||||
// console.log(data);
|
||||
// callback(JSON.parse(data));
|
||||
resolve(JSON.parse(data));
|
||||
|
||||
const rawJSON = JSON.parse(data);
|
||||
|
||||
const rv = rawJSON.contents.quotes[0];
|
||||
resolve(rv);
|
||||
});
|
||||
}).end();
|
||||
});
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
/*
|
||||
|
||||
*/
|
||||
|
||||
2398
package-lock.json
generated
2398
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
22
package.json
22
package.json
@ -14,24 +14,26 @@
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"cheerio": "^1.0.0-rc.2",
|
||||
"cloudant": "^1.8.0",
|
||||
"dateformat": "^3.0.2",
|
||||
"cloudant": "^1.10.0",
|
||||
"dateformat": "^3.0.3",
|
||||
"elapsed": "0.0.7",
|
||||
"fitbit-oauth2": "0.0.1",
|
||||
"forecast.io": "0.0.11",
|
||||
"ftse": "^1.0.6",
|
||||
"jsonfile": "^4.0.0",
|
||||
"lodash": "^4.17.4",
|
||||
"log4js": "^2.3.10",
|
||||
"lodash": "^4.17.11",
|
||||
"log4js": "^2.11.0",
|
||||
"node-cron": "^1.2.1",
|
||||
"node-localstorage": "^1.3.0",
|
||||
"pug": "^2.0.0-rc.4",
|
||||
"request": "^2.83.0",
|
||||
"node-localstorage": "^1.3.1",
|
||||
"pug": "^2.0.3",
|
||||
"request": "^2.88.0",
|
||||
"string": "^3.3.3",
|
||||
"sugar": "^2.0.4",
|
||||
"sugar-date": "^2.0.4",
|
||||
"sugar": "^2.0.6",
|
||||
"sugar-date": "^2.0.6",
|
||||
"ultrases": "^0.1.3",
|
||||
"xmljson": "^0.2.0"
|
||||
},
|
||||
"engines" : { "node" : ">= 8.1.0" }
|
||||
"engines": {
|
||||
"node": ">= 8.1.0"
|
||||
}
|
||||
}
|
||||
|
||||
31
test.js
Normal file
31
test.js
Normal file
@ -0,0 +1,31 @@
|
||||
const cheerio = require('cheerio');
|
||||
|
||||
const fs = require('fs');
|
||||
const jsonfile = require('jsonfile');
|
||||
|
||||
const logger = require('log4js').getLogger('history');
|
||||
|
||||
logger.level = 'debug';
|
||||
|
||||
function run() {
|
||||
const output = [];
|
||||
const h = fs.readFileSync('tests/data/hist001.html');
|
||||
|
||||
const $ = cheerio.load(h);
|
||||
const bigBlock = $('#chm-exhibit-content > article > div > div');
|
||||
|
||||
const rows = $(bigBlock).find('div.chm-exhibit-timeline-decade-row');
|
||||
|
||||
rows.each((i, item) => {
|
||||
const title = $(item).find('.chm-tdih-entry-title').text();
|
||||
const content = $(item).find('.chm-tdih-entry-content').text();
|
||||
|
||||
logger.debug(title);
|
||||
|
||||
output.push({ title, content });
|
||||
});
|
||||
|
||||
logger.debug(JSON.stringify(output));
|
||||
}
|
||||
|
||||
run();
|
||||
279
tests/data/hist001.html
Normal file
279
tests/data/hist001.html
Normal file
@ -0,0 +1,279 @@
|
||||
<html lang="en"><head>
|
||||
<!-- Google Tag Manager -->
|
||||
|
||||
<!-- End Google Tag Manager -->
|
||||
<meta charset="UTF-8">
|
||||
|
||||
<title>January 1: The IEEE Computer Group Becomes the IEEE Computer Society | This Day in History | Computer History Museum</title>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!--[if lte IE 8]>
|
||||
|
||||
<![endif]-->
|
||||
<!--[if gt IE 8]>
|
||||
|
||||
<![endif]-->
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<!-- Google Tag Manager (noscript) -->
|
||||
<noscript><iframe
|
||||
height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
|
||||
<!-- End Google Tag Manager (noscript) -->
|
||||
<div id="chm-exhibit-container">
|
||||
<header id="chm-exhibit-header" class="pure-g">
|
||||
<a id="chm-exhibit-logo" href="/tdih/">
|
||||
<img class="pure-img" alt="This Day in History | Computer History Museum">
|
||||
</a>
|
||||
</header>
|
||||
|
||||
|
||||
<nav id="chm-exhibit-nav-bar">
|
||||
<div id="today" class="chm-exhibit-nav-bar-item"><a class="chm-exhibit-nav-bar-link" href="/tdih/">Today</a></div>
|
||||
<div id="January" class="chm-exhibit-nav-bar-item"><a class="chm-exhibit-nav-bar-link" href="/tdih/january/">January</a></div>
|
||||
<div id="February" class="chm-exhibit-nav-bar-item"><a class="chm-exhibit-nav-bar-link" href="/tdih/february/">February</a></div>
|
||||
<div id="March" class="chm-exhibit-nav-bar-item"><a class="chm-exhibit-nav-bar-link" href="/tdih/march/">March</a></div>
|
||||
<div id="April" class="chm-exhibit-nav-bar-item"><a class="chm-exhibit-nav-bar-link" href="/tdih/april/">April</a></div>
|
||||
<div id="May" class="chm-exhibit-nav-bar-item"><a class="chm-exhibit-nav-bar-link" href="/tdih/may/">May</a></div>
|
||||
<div id="June" class="chm-exhibit-nav-bar-item"><a class="chm-exhibit-nav-bar-link" href="/tdih/june/">June</a></div>
|
||||
<div id="July" class="chm-exhibit-nav-bar-item"><a class="chm-exhibit-nav-bar-link" href="/tdih/july/">July</a></div>
|
||||
<div id="August" class="chm-exhibit-nav-bar-item"><a class="chm-exhibit-nav-bar-link" href="/tdih/august/">August</a></div>
|
||||
<div id="September" class="chm-exhibit-nav-bar-item"><a class="chm-exhibit-nav-bar-link" href="/tdih/september/">September</a></div>
|
||||
<div id="October" class="chm-exhibit-nav-bar-item"><a class="chm-exhibit-nav-bar-link" href="/tdih/october/">October</a></div>
|
||||
<div id="November" class="chm-exhibit-nav-bar-item"><a class="chm-exhibit-nav-bar-link" href="/tdih/november/">November</a></div>
|
||||
<div id="December" class="chm-exhibit-nav-bar-item"><a class="chm-exhibit-nav-bar-link" href="/tdih/december/">December</a></div>
|
||||
<div id="search" class="chm-exhibit-nav-bar-item"><a class="chm-exhibit-nav-bar-link" href="/tdih/search/"><i class="fa fa-search" aria-hidden="true"></i></a></div>
|
||||
</nav>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<main id="chm-exhibit-content" class="pure-g slidr" role="main">
|
||||
<article class="chm-exhibit-content pure-u-1-1">
|
||||
<div class="content pure-g clearfix">
|
||||
<h1 id="timeline" class="chm-exhibit-content-header">What Happened on January 1st</h1>
|
||||
|
||||
<div class="chm-exhibit-container pure-u-1-1 clearfix chm-mt-24">
|
||||
<div id="the-ieee-computer-group-becomes-the-ieee-computer-society" class="chm-exhibit-timeline-decade-row pure-g clearfix">
|
||||
<div class="pure-u-5-12 pull-left clearfix chm-mb-24">
|
||||
<div class="media day border day clearfix">
|
||||
<div class="image day">
|
||||
<img class="img-responsive link" alt="">
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="pure-u-1-24 pull-left"> </div>
|
||||
|
||||
|
||||
<div class="pure-u-13-24 pull-right">
|
||||
<span class="chm-tdih-entry-date">january 1, 1971</span>
|
||||
<span class="chm-tdih-entry-title">The IEEE Computer Group Becomes the IEEE Computer Society</span>
|
||||
<div class="chm-tdih-entry-content">The IEEE Computer Group formally changes its name to the IEEE Computer Society (CS). The origin of both of these entities was the foundation of the subcommittee on Large-Scale Computing Devices by the American Institute of Electrical Engineers in 1946.
|
||||
Today, the IEEE CS is one of the largest of several dozen technical societies of IEEE, which has total membership of more than 430,000 in over 160 countries as of 2014.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div id="eckert-and-mauchly-sign-a-contract-to-build-the-edvac" class="chm-exhibit-timeline-decade-row pure-g clearfix">
|
||||
<div class="pure-u-5-12 pull-left clearfix chm-mb-24">
|
||||
<div class="media day border day clearfix">
|
||||
<div class="image day">
|
||||
<img class="img-responsive link" alt="The EDVAC">
|
||||
</div>
|
||||
|
||||
<div class="captions clearfix">
|
||||
<span>The EDVAC</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="pure-u-1-24 pull-left"> </div>
|
||||
|
||||
|
||||
<div class="pure-u-13-24 pull-right">
|
||||
<span class="chm-tdih-entry-date">january 1, 1945</span>
|
||||
<span class="chm-tdih-entry-title">Eckert and Mauchly Sign a Contract to Build the EDVAC</span>
|
||||
<div class="chm-tdih-entry-content">Eckert and Mauchly sign a contract to build the EDVAC (Electronic Discrete Variable Computer), the first general-purpose electronic digital-stored program computer to be designed. Even before the ENIAC had been unveiled in 1946, Eckert and Mauchly were already thinking about their next machine. In June 1945, John von Neumann, who took an active part in the design discussions, made a key contribution to the understanding and development of computer architecture in his unpublished report titled First Draft of a Report on the EDVAC. <p>The EDVAC was completed only in 1952, long after Eckert and Mauchly had left the University of Pennsylvania. The computer, which was heavily modified from its original design, had been used for ballistic and satellite calculations. The EDVAC was shut off in January 1963.</p></div>
|
||||
</div>
|
||||
</div>
|
||||
<div id="mailufterls-developer-heinz-zemanek-born" class="chm-exhibit-timeline-decade-row pure-g clearfix">
|
||||
<div class="pure-u-5-12 pull-left clearfix chm-mb-24">
|
||||
<div class="media day border day clearfix">
|
||||
<div class="image day">
|
||||
<img class="img-responsive link" alt="Heinz Zemanek">
|
||||
</div>
|
||||
|
||||
<div class="captions clearfix">
|
||||
<span>Heinz Zemanek</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="pure-u-1-24 pull-left"> </div>
|
||||
|
||||
|
||||
<div class="pure-u-13-24 pull-right">
|
||||
<span class="chm-tdih-entry-date">january 1, 1920</span>
|
||||
<span class="chm-tdih-entry-title">Mailufterl's Developer Heinz Zemanek Born</span>
|
||||
<div class="chm-tdih-entry-content">Heinz Zemanek, one of the European pioneers in computer technology, is born in Vienna, Austria. He studied at the University of Technology Vienna and received a doctorate degree in 1951. From 1954 to 1959 Zemanek gathered a group of students to develop the Mailüfterl, one of the earliest fully transistorized computers in Europe. Later, from 1961 to 1975, Zemanek was Director of the IBM Laboratory Vienna which was established for his team. During this period Zemanek designed the PL/I programming language. The formal definition was written in the Vienna Definition Language, which was later extended to the Vienna Definition Method (VDL and VDM). From 1968 to 1971 he founded the Austrian Computer Society and was President of the International Federation for Information Processing (IFIP). In 1976 Professor Zemanek was appointed IBM Fellow. </div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
</article>
|
||||
</main>
|
||||
|
||||
|
||||
|
||||
<div id="slider" class="left">
|
||||
<div style="display: block; width: 280px; height: 100%; position: relative;">
|
||||
<div id="chm-slider-left-menu-content" class="slider" style="overflow-x: hidden; white-space: nowrap;">
|
||||
<ul id="slider-left-menu">
|
||||
<li><a class="chm-logo" href="/"><img alt="Computer History Museum"></a></li>
|
||||
<li><a class="" href="/visit/">Visit Us</a></li>
|
||||
<li><a class="" href="/exhibits/">Exhibits</a>
|
||||
<ul id="slider-left-menu-exhibits">
|
||||
<li class="slider-left-menu-subtitle"><h3>Online Exhibits</h3></li>
|
||||
<li id="revolution"><a href="/revolution/">Revolution</a></li>
|
||||
<li id="makesofwtare"><a href="/makesoftware/">Make Software: Change the World!</a></li>
|
||||
<li id="fellowawards"><a href="/fellowawards/hall/">Hall of Fellows</a></li>
|
||||
<li id="internethistory"><a href="/internethistory/">Internet History 1962 to 1992</a></li>
|
||||
<li id="chess"><a href="/chess/">Mastering the Game</a></li>
|
||||
<li id="microprocessors"><a href="/microprocessors/">Microprocessors 1971 to 1996</a></li>
|
||||
<li id="pdp-1"><a href="/pdp-1/">PDP-1 Restoration Project</a></li>
|
||||
<li id="brochures"><a href="/brochures/">Selling the Computer Revolution</a></li>
|
||||
<li id="babbage"><a href="/babbage/">The Babbage Engine</a></li>
|
||||
<li id="siliconengine"><a href="/siliconengine/">The Silicon Engine</a></li>
|
||||
<li id="storageengine"><a href="/storageengine/">The Storage Engine</a></li>
|
||||
<li id="tdih" class="active"><a href="/tdih/">This Day in History</a></li>
|
||||
<li id="timeline"><a href="/timeline/">Timeline of Computer History</a></li>
|
||||
<li id="virtualvisiblestorage"><a href="/visiblestorage/">Visible Storage</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a class="" href="/events/upcoming/">Events</a></li>
|
||||
<li><a class="" href="/education/">Education</a></li>
|
||||
<li><a class="" href="/collections/">Collections</a></li>
|
||||
<li><a class="" href="/centers/">Centers</a></li>
|
||||
<li><a class="" href="/atchm/">@chm</a></li>
|
||||
</ul>
|
||||
|
||||
<div id="slider-social-media-container" class="pure-g">
|
||||
<div class="slider-social-media">
|
||||
<a class="slider-social-media facebook" href="https://www.facebook.com/computerhistory/" target="new">
|
||||
<img alt="Like Us on Facebook" hidden="" style="display: none !important;"> </a>
|
||||
</div>
|
||||
<div class="slider-social-media">
|
||||
<a class="slider-social-media twitter" href="https://twitter.com/ComputerHistory" target="new">
|
||||
<img alt="Follow Us on Twitter" hidden="" style="display: none !important;"> </a>
|
||||
</div>
|
||||
<div class="slider-social-media">
|
||||
<a class="slider-social-media youtube" href="https://www.youtube.com/user/ComputerHistory" target="new">
|
||||
<img alt="" hidden="" style="display: none !important;"> </a>
|
||||
</div>
|
||||
<div class="slider-social-media">
|
||||
<a class="slider-social-media instagram" href="https://www.instagram.com/computerhistory/" target="new">
|
||||
<img alt="" hidden="" style="display: none !important;"> </a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="chm-slider-left-menu-controls">
|
||||
<div id="chm-slider-menu-open" class="c-hamburger c-hamburger--htx"><span>toggle menu</span></div>
|
||||
<div id="chm-slider-menu-logo" class="logo"><img alt="Computer History Museum"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<!--// Footer Content -->
|
||||
<footer id="chm-exhibit-footer" class="">
|
||||
© Computer History Museum
|
||||
</footer>
|
||||
<!-- Footer Content //-->
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- -->
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<!-- new sidr file -->
|
||||
<div id="sidr" style="display: none;">
|
||||
<ul id="sidr-left-menu">
|
||||
<li><a class="chm-logo" href="/"><img alt="Computer History Museum"></a></li>
|
||||
<li><a class="" href="/visit/">Visit Us</a></li>
|
||||
<li><a class="" href="/exhibits/">Exhibits</a>
|
||||
<ul id="sidr-left-menu-exhibits">
|
||||
<li class="sidr-left-menu-subtitle"><h3>Online Exhibits</h3></li>
|
||||
<li id="revolution"><a href="/revolution/">Revolution</a></li>
|
||||
<li id="makesoftware"><a href="/makesoftware/">Make Software: Change the World!</a></li>
|
||||
<li id="fellowawards"><a href="/fellowawards/hall/">Hall of Fellows</a></li>
|
||||
<li id="internethistory"><a href="/internethistory/">Internet History 1962 to 1992</a></li>
|
||||
<li id="chess"><a href="/chess/">Mastering the Game</a></li>
|
||||
<li id="microprocessors"><a href="/microprocessors/">Microprocessors 1971 to 1996</a></li>
|
||||
<li id="pdp-1"><a href="/pdp-1/">PDP-1 Restoration Project</a></li>
|
||||
<li id="brochures"><a href="/brochures/">Selling the Computer Revolution</a></li>
|
||||
<li id="babbage"><a href="/babbage/">The Babbage Engine</a></li>
|
||||
<li id="siliconengine"><a href="/siliconengine/">The Silicon Engine</a></li>
|
||||
<li id="storageengine"><a href="/storageengine/">The Storage Engine</a></li>
|
||||
<li id="tdih"><a href="/tdih/">This Day in History</a></li>
|
||||
<li id="timeline"><a href="/timeline/">Timeline of Computer History</a></li>
|
||||
<li id="virtualvisiblestorage"><a href="/virtualvisiblestorage/">Visible Storage</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a class="" href="/events/upcoming/">Events</a></li>
|
||||
<li><a class="" href="/education/">Education</a></li>
|
||||
<li><a class="" href="/collections/">Collections</a></li>
|
||||
<li><a class="" href="/centers/">Centers</a></li>
|
||||
<li><a class="" href="/atchm/">@chm</a></li>
|
||||
</ul>
|
||||
|
||||
<div id="sidr-social-media-container" class="pure-g">
|
||||
<div class="sidr-social-media">
|
||||
<a class="sidr-social-media facebook" a="" href="https://www.facebook.com/computerhistory/" target="new"><img alt="Like Us on Facebook" hidden="" style="display: none !important;">
|
||||
</a>
|
||||
</div>
|
||||
<div class="sidr-social-media">
|
||||
<a class="sidr-social-media twitter" a="" href="https://twitter.com/ComputerHistory" target="new"><img alt="Follow Us on Twitter" hidden="" style="display: none !important;">
|
||||
</a>
|
||||
</div>
|
||||
<div class="sidr-social-media">
|
||||
<a class="sidr-social-media youtube" a="" href="https://www.youtube.com/user/ComputerHistory" target="new"><img alt="" hidden="" style="display: none !important;">
|
||||
</a>
|
||||
</div>
|
||||
<div class="sidr-social-media">
|
||||
<a class="sidr-social-media instagram" a="" href="https://www.instagram.com/computerhistory/" target="new"><img alt="" hidden="" style="display: none !important;">
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</div><a id="scrollUp" href="#top" style="display: none; position: fixed; z-index: 2147483647;"></a></body></html>
|
||||
19
todayV2.js
19
todayV2.js
@ -460,12 +460,21 @@ module.exports = {
|
||||
}
|
||||
};
|
||||
|
||||
function doGC() {
|
||||
if (global.gc) {
|
||||
global.gc();
|
||||
} else {
|
||||
console.log('Garbage collection unavailable. use --expose-gc '
|
||||
+ 'when launching node to enable forced garbage collection.');
|
||||
}
|
||||
}
|
||||
|
||||
setTimeout(function () {
|
||||
loadData();
|
||||
if (runable())
|
||||
module.exports.preLoadToday();
|
||||
|
||||
// Module.exports.preLoadToday();
|
||||
// module.exports.preLoadToday();
|
||||
}, 5000);
|
||||
|
||||
setTimeout(function () {
|
||||
@ -473,6 +482,7 @@ setTimeout(function () {
|
||||
mdMailer.sendEmailV1(todayCache, __dirname);
|
||||
saveToDB(todayCache);
|
||||
saveData();
|
||||
doGC();
|
||||
}, 45000);
|
||||
|
||||
setInterval(function () {
|
||||
@ -501,3 +511,10 @@ cron.schedule('30 7 * * *', function () {
|
||||
return -1;
|
||||
});
|
||||
|
||||
|
||||
cron.schedule('45 7 * * *', function () {
|
||||
mdMailer.sendEmailV1(todayCache, __dirname);
|
||||
// saveToDB(todayCache);
|
||||
// Console.log('tick');
|
||||
return -1;
|
||||
});
|
||||
|
||||
Loading…
Reference in New Issue
Block a user