dataProvider.js

var async = require('async')
, fs = require('fs')
, path = require('path')
, rest = require('./rest')
, objectUtil = require('./object-util')
, defaultConfig = require('./config').getDefaultConfig();




var grabJSON = exports.grabJSON = function(version) {
 var nodeVersion = version || process.versions.node;
 var getOptions = function(path) {
 var versionString = 'v' + nodeVersion;
 return {
 host: 'nodejs.org',
 method: 'GET',
 path: '/docs/' + versionString + (path || '/api/index.json')
 };
 }
 rest.getResponse(getOptions(), function(response, output) {
 if(response.statusCode == 200) {
 var jsonObj = JSON.parse(output);
 var desc = jsonObj.desc;

 var jsonFileNames = [];
 for(var i=0, item; item= desc[i]; i++) {
 if(!item.text) continue;

 var matchResult = item.text.match(/\((\w+)\.\w+\)/);
 if(matchResult) {
 jsonFileNames.push(matchResult[1].concat('.json'));
 }
 }

 var apiList = {};
 var JSON_DIR = __dirname
 + '/public/json/'
 + nodeVersion;


 async.waterfall([
 function(callback) {
 fs.exists(JSON_DIR, function(exists) {
 callback(null, exists);
 });
 },
 function(exists, callback) {
 if(exists) {
 callback(null);
 } else {
 fs.mkdir(JSON_DIR, callback);
 }
 }
 ], function(err) {
 if(!err) {
 var q = async.queue(function(task, callback) {
 var path = '/api/'.concat(task);
 rest.getResponse(getOptions(path),
 function(response, output) {
 if(response.statusCode == 200) {
 var jsonObj = JSON.parse(output);
 apiList[task] = objectUtil
 .extractObjectKeyPath(jsonObj, 'textRaw');
 fs.writeFile(JSON_DIR + '/' + task,
 JSON.stringify(jsonObj), 'utf-8', function(err) {
 if(err) {
 console.log(err.stack);
 }
 });

 callback();
 }
 });
 }, jsonFileNames.length);

 q.drain = function() {
 var stringData = JSON.stringify(apiList);
 fs.writeFile(JSON_DIR + '/mydata.json',
 stringData, 'utf-8', function(err) {
 if(!err) {
 console.log('finished generate mydata.json');
 } else {
 console.log(err.stack);
 }
 });
 };

 q.push(jsonFileNames, function(err) {

 });
 }
 });
 }
 });
};

grabJSON(defaultConfig.API_VERSION);

process.on('uncaughtException', function(err) {
 console.log(err.stack);
});