Converted recipe handling to use async IO

Part of the preparation for supporting reading recipes and ingredients
over HTTP
This commit is contained in:
Jeremy Ruston 2011-11-28 13:47:38 +00:00
parent a2831eb203
commit 5314fda2ca
6 changed files with 133 additions and 42 deletions

View File

@ -10,6 +10,7 @@ var filename = process.argv[2];
var store = new TiddlyWiki();
var theRecipe = new Recipe(store,filename);
var theRecipe = new Recipe(store,filename,function() {
process.stdout.write(theRecipe.cook());
});
process.stdout.write(theRecipe.cook());

23
js/FileRetriever.js Normal file
View File

@ -0,0 +1,23 @@
/*
FileRetriever can asynchronously retrieve files from HTTP URLs or the local file system. It incorporates
throttling so that we don't get error EMFILE "Too many open files".
*/
var fs = require("fs"),
utils = require("./Utils.js");
var FileRetriever = exports;
var fileRequestQueue = utils.queue(function(task,callback) {
fs.readFile(task.filepath,"utf8", function(err,data) {
callback(err,data);
});
},10);
// Retrieve a file given a filepath specifier and a context path. If the filepath isn't an absolute
// filepath or an absolute URL, then it is interpreted relative to the context path, which can also be
// a filepath or a URL. On completion, the callback function is called as callback(err,data)
FileRetriever.retrieveFile = function(filepath,contextPath,callback) {
fileRequestQueue.push({filepath: filepath},callback);
}

View File

@ -32,33 +32,48 @@ var Tiddler = require("./Tiddler.js").Tiddler,
tiddlerOutput = require("./TiddlerOutput.js"),
utils = require("./Utils.js"),
TiddlyWiki = require("./TiddlyWiki.js").TiddlyWiki,
retrieveFile = require("./FileRetriever.js").retrieveFile,
fs = require("fs"),
path = require("path"),
util = require("util");
// Create a new Recipe object from the specified recipe file, storing the tiddlers in a specified TiddlyWiki store
var Recipe = function(store,filepath) {
// Create a new Recipe object from the specified recipe file, storing the tiddlers in a specified TiddlyWiki store. Invoke
// the callback function when all of the referenced tiddlers and recipes have been loaded successfully
var Recipe = function(store,filepath,callback) {
this.store = store; // Save a reference to the store
this.ingredients = {}; // Hashmap of array of ingredients
this.callback = callback;
this.fetchCount = 0;
this.readRecipe(filepath); // Read the recipe file
}
// Specialised configuration and handlers for particular ingredient markers
var specialMarkers = {
shadow: {
readIngredientPostProcess: function(fields) {
// Add ".shadow" to the name of shadow tiddlers
fields.title = fields.title + ".shadow";
return fields;
}
// The fetch counter is used to keep track of the number of asynchronous requests outstanding
Recipe.prototype.incFetchCount = function() {
this.fetchCount++;
}
// When the fetch counter reaches zero, all the results are in, so invoke the recipe callback
Recipe.prototype.decFetchCount = function() {
if(--this.fetchCount === 0) {
this.callback();
}
};
}
// Process the contents of a recipe file
Recipe.prototype.readRecipe = function(filepath) {
var dirname = path.dirname(filepath),
me = this;
fs.readFileSync(filepath,"utf8").split("\n").forEach(function(line) {
this.incFetchCount();
retrieveFile(filepath, null, function(err, data) {
if (err) throw err;
me.processRecipe(data,dirname);
me.decFetchCount();
});
}
Recipe.prototype.processRecipe = function (data,dirname) {
var me = this;
data.split("\n").forEach(function(line) {
var p = line.indexOf(":");
if(p !== -1) {
var marker = line.substr(0, p).trim(),
@ -66,11 +81,19 @@ Recipe.prototype.readRecipe = function(filepath) {
if(marker === "recipe") {
me.readRecipe(path.resolve(dirname,value));
} else {
var fields = me.readIngredient(dirname,value),
postProcess = me.readIngredientPostProcess[marker];
if(postProcess)
fields = postProcess(fields);
me.addIngredient(marker,fields);
if(!(marker in me.ingredients)) {
me.ingredients[marker] = [];
}
var ingredientLocation = me.ingredients[marker].length;
me.ingredients[marker][ingredientLocation] = null;
me.readIngredient(dirname,value,function(fields) {
var postProcess = me.readIngredientPostProcess[marker];
if(postProcess)
fields = postProcess(fields);
var ingredientTiddler = new Tiddler(fields);
me.store.addTiddler(ingredientTiddler);
me.ingredients[marker][ingredientLocation] = ingredientTiddler;
});
}
}
});
@ -85,32 +108,35 @@ Recipe.prototype.readIngredientPostProcess = {
}
};
Recipe.prototype.addIngredient = function(marker,tiddlerFields) {
var ingredientTiddler = new Tiddler(tiddlerFields);
this.store.addTiddler(ingredientTiddler);
if(marker in this.ingredients) {
this.ingredients[marker].push(ingredientTiddler);
} else {
this.ingredients[marker] = [ingredientTiddler];
}
}
// Read an ingredient file and return it as a hashmap of tiddler fields. Also read the .meta file, if present
Recipe.prototype.readIngredient = function(dirname,filepath) {
var fullpath = path.resolve(dirname,filepath),
Recipe.prototype.readIngredient = function(dirname,filepath,callback) {
var me = this,
fullpath = path.resolve(dirname,filepath),
extname = path.extname(filepath),
basename = path.basename(filepath,extname),
fields = {
title: basename
};
me.incFetchCount();
// Read the tiddler file
fields = tiddlerInput.parseTiddler(fs.readFileSync(fullpath,"utf8"),extname,fields);
// Check for the .meta file
var metafile = fullpath + ".meta";
if(path.existsSync(metafile)) {
fields = tiddlerInput.parseMetaDataBlock(fs.readFileSync(metafile,"utf8"),fields);
}
return fields;
retrieveFile(fullpath,null,function(err,data) {
if (err) throw err;
fields = tiddlerInput.parseTiddler(data,extname,fields);
// Check for the .meta file
var metafile = fullpath + ".meta";
me.incFetchCount();
retrieveFile(metafile,null,function(err,data) {
if(err && err.code !== 'ENOENT') {
throw err;
}
if(!err) {
fields = tiddlerInput.parseMetaDataBlock(data,fields);
}
callback(fields);
me.decFetchCount();
});
me.decFetchCount();
});
}
// Return a string of the cooked recipe
@ -128,7 +154,6 @@ Recipe.prototype.cook = function() {
out.push(line);
}
});
// out.push("\nRecipe:\n" + util.inspect(this.ingredients,false,4));
return out.join("\n");
}

View File

@ -72,3 +72,42 @@ utils.htmlDecode = function(s)
return s.replace(/&lt;/mg,"<").replace(/&gt;/mg,">").replace(/&quot;/mg,"\"").replace(/&amp;/mg,"&");
};
// Adapted from async.js, https://github.com/caolan/async
// Creates a queue of tasks for an asyncronous worker function with a specified maximum number of concurrent operations.
// q = utils.queue(function(taskData,callback) {
// fs.readFile(taskData.filename,"uft8",function(err,data) {
// callback(err,data);
// });
// });
// q.push(taskData,callback) is used to queue a new task
utils.queue = function(worker, concurrency) {
var workers = 0;
var q = {
tasks: [],
concurrency: concurrency,
push: function (data, callback) {
q.tasks.push({data: data, callback: callback});
process.nextTick(q.process);
},
process: function () {
if (workers < q.concurrency && q.tasks.length) {
var task = q.tasks.shift();
workers += 1;
worker(task.data, function () {
workers -= 1;
if (task.callback) {
task.callback.apply(task, arguments);
}
q.process();
});
}
},
length: function () {
return q.tasks.length;
},
running: function () {
return workers;
}
};
return q;
};

View File

@ -12,11 +12,12 @@ var TiddlyWiki = require("./js/TiddlyWiki.js").TiddlyWiki,
var filename = process.argv[2];
http.createServer(function (request, response) {
http.createServer(function(request, response) {
response.writeHead(200, {"Content-Type": "text/html"});
var store = new TiddlyWiki();
var theRecipe = new Recipe(store,filename);
response.end(theRecipe.cook(), "utf-8");
var theRecipe = new Recipe(store,filename,function() {
response.end(theRecipe.cook(), "utf-8");
});
}).listen(8000);
sys.puts("Server running at http://127.0.0.1:8000/");

View File

@ -0,0 +1,2 @@
tags: one two three four five
modifier: jermolene