From 6bbc6c0dacf93a920297a19f1d40a817b7c6efa5 Mon Sep 17 00:00:00 2001
From: Jeremy Ruston <jeremy@osmosoft.com>
Date: Tue, 29 Nov 2011 18:27:03 +0000
Subject: [PATCH] Added support for retrieving ingredients over HTTP

This makes it possible to directly reference recipes and tiddlers
stored on GitHub, for example.
---
 js/FileRetriever.js                           | 48 +++++++++++++++++--
 js/Recipe.js                                  |  2 +-
 .../data/recipes/newtiddlerswithgithub.recipe |  2 +
 3 files changed, 48 insertions(+), 4 deletions(-)
 create mode 100644 test/data/recipes/newtiddlerswithgithub.recipe

diff --git a/js/FileRetriever.js b/js/FileRetriever.js
index a7cfafc0b..16b27acb8 100644
--- a/js/FileRetriever.js
+++ b/js/FileRetriever.js
@@ -5,6 +5,10 @@ throttling so that we don't get error EMFILE "Too many open files".
 
 var fs = require("fs"),
 	path = require("path"),
+	url = require("url"),
+	util = require("util"),
+	http = require("http"),
+	https = require("https"),
 	utils = require("./Utils.js");
 
 var FileRetriever = exports;
@@ -12,15 +16,53 @@ var FileRetriever = exports;
 var fileRequestQueue = utils.queue(function(task,callback) {
 	fs.readFile(task.filepath,"utf8", function(err,data) {
 		callback(err,data);
+console.error("Retrieved " + task.filepath);
 	});
 },10);
 
+var httpRequestQueue = utils.queue(function(task,callback) {
+	var opts = url.parse(task.url);
+	var httpLib = opts.protocol === "http:" ? http : https;
+	var request = httpLib.get(opts,function(res) {
+		if(res.statusCode != 200) {
+			var err = new Error("HTTP error");
+			err.code = res.statusCode.toString();
+			callback(err);
+		} else {
+			var data = [];
+			res.on("data", function(chunk) {
+				data.push(chunk)
+			});
+			res.on("end", function() {
+				callback(null,data.join(""));
+console.error("Retrieved " + task.url);
+			});
+		}
+	});
+	request.addListener("error", function(err) {
+	    callback(err);
+	});
+	request.end();
+},4);
+
 // Retrieve a file given a filepath specifier and a context path. If the filepath isn't an absolute
 // filepath or an absolute URL, then it is interpreted relative to the context path, which can also be
 // a filepath or a URL. It returns the final path used to reach the file. On completion, the callback
 // function is called as callback(err,data)
 FileRetriever.retrieveFile = function(filepath,contextPath,callback) {
-	var newpath = path.resolve(path.dirname(contextPath),filepath);
-	fileRequestQueue.push({filepath: newpath},callback);
-	return newpath;
+	var httpRegExp = /^(https?:\/\/)/gi,
+		newpath,
+		filepathIsHttp = httpRegExp.test(filepath),
+		contextPathIsHttp = httpRegExp.test(contextPath);
+	if(contextPathIsHttp || filepathIsHttp) {
+		// If we've got a full HTTP URI then we're good to go
+		newpath = url.resolve(contextPath,filepath);
+		httpRequestQueue.push({url: newpath},callback);
+		return newpath;
+	} else {
+		// It's a file requested in a file context
+		newpath = path.resolve(path.dirname(contextPath),filepath);
+		fileRequestQueue.push({filepath: newpath},callback);
+		return newpath;
+	}
 }
diff --git a/js/Recipe.js b/js/Recipe.js
index 0edb5b785..dc011f5cf 100755
--- a/js/Recipe.js
+++ b/js/Recipe.js
@@ -124,7 +124,7 @@ Recipe.prototype.readIngredient = function(filepath,contextPath,callback) {
 		var metafile = filepath + ".meta";
 		me.incFetchCount();
 		retrieveFile(metafile,contextPath,function(err,data) {
-			if(err && err.code !== 'ENOENT') {
+			if(err && err.code !== "ENOENT" && err.code !== "404") {
 				throw err;
 			}
 			if(!err) {
diff --git a/test/data/recipes/newtiddlerswithgithub.recipe b/test/data/recipes/newtiddlerswithgithub.recipe
new file mode 100644
index 000000000..7c18ea313
--- /dev/null
+++ b/test/data/recipes/newtiddlerswithgithub.recipe
@@ -0,0 +1,2 @@
+recipe: https://raw.github.com/TiddlyWiki/tiddlywiki/master/tiddlywikinonoscript.html.recipe
+recipe: ../tiddlywiki.com/index.html.recipe