From 80d71d7bf44bd1848a43f5dd736dd193b767c09d Mon Sep 17 00:00:00 2001 From: Jeremy Ruston Date: Sun, 11 Dec 2011 18:28:09 +0000 Subject: [PATCH] Major refactoring of how wiki text parsing and rendering is packaged --- js/Recipe.js | 17 +-- js/TextProcessors.js | 26 ++++ js/Tiddler.js | 23 ---- js/WikiStore.js | 26 +++- js/WikiTextParser.js | 38 +++--- js/WikiTextProcessor.js | 30 +++++ js/WikiTextRenderer.js | 28 ++--- js/WikiTextRules.js | 14 +-- readme.md | 260 +++++++++++++++++++++++++++++++++++++++- tiddlywiki.js | 26 ++-- wikitest.js | 16 ++- 11 files changed, 411 insertions(+), 93 deletions(-) create mode 100644 js/TextProcessors.js create mode 100644 js/WikiTextProcessor.js diff --git a/js/Recipe.js b/js/Recipe.js index 630eef59b..76fdb72dd 100755 --- a/js/Recipe.js +++ b/js/Recipe.js @@ -50,7 +50,6 @@ At this point tiddlers are placed in the store so that they can be referenced by "use strict"; var Tiddler = require("./Tiddler.js").Tiddler, - WikiTextRenderer = require("./WikiTextRenderer").WikiTextRenderer, utils = require("./Utils.js"), retrieveFile = require("./FileRetriever.js").retrieveFile, fs = require("fs"), @@ -63,6 +62,7 @@ var Recipe = function(options,callback) { this.filepath = options.filepath; this.store = options.store; this.tiddlerConverters = options.tiddlerConverters; + this.textProcessors = options.textProcessors; this.callback = callback; this.recipe = []; this.markers = {}; @@ -280,7 +280,7 @@ Recipe.tiddlerOutputter = { } }, title: function(out,tiddlers) { - out.push(this.renderTiddler("WindowTitle","text/plain")); + out.push(this.store.renderTiddler("text/plain","WindowTitle")); } }; @@ -291,13 +291,13 @@ Recipe.prototype.cookRss = function() numRssItems = 20, s = [], d = new Date(), - u = this.renderTiddler("SiteUrl","text/plain"), + u = this.store.renderTiddler("text/plain","SiteUrl"), encodeTiddlyLink = function(title) { return title.indexOf(" ") == -1 ? title : "[[" + title + "]]"; }, tiddlerToRssItem = function(tiddler,uri) { var s = "" + utils.htmlEncode(tiddler.fields.title) + "\n"; - s += "" + utils.htmlEncode(me.renderTiddler(tiddler.fields.title,"text/plain")) + "\n"; + s += "" + utils.htmlEncode(me.store.renderTiddler("text/plain",tiddler.fields.title)) + "\n"; var i; if(tiddler.fields.tags) { for(i=0; i"); s.push(""); s.push(""); - s.push("" + utils.htmlEncode(this.renderTiddler("SiteTitle","text/plain")) + ""); + s.push("" + utils.htmlEncode(this.store.renderTiddler("text/plain","SiteTitle")) + ""); if(u) s.push("" + utils.htmlEncode(u) + ""); - s.push("" + utils.htmlEncode(this.renderTiddler("SiteSubtitle","text/plain")) + ""); + s.push("" + utils.htmlEncode(this.store.renderTiddler("text/plain","SiteSubtitle")) + ""); //s.push("" + config.locale + ""); s.push("" + d.toUTCString() + ""); s.push("" + d.toUTCString() + ""); @@ -358,10 +358,5 @@ Recipe.prototype.cookRss = function() return s.join("\n"); }; -Recipe.prototype.renderTiddler = function(title,type) { - var r = new WikiTextRenderer(this.store.getTiddler(title).getParseTree(),this.store,title); - return r.render(type); -}; - exports.Recipe = Recipe; diff --git a/js/TextProcessors.js b/js/TextProcessors.js new file mode 100644 index 000000000..6dace7fc2 --- /dev/null +++ b/js/TextProcessors.js @@ -0,0 +1,26 @@ +/*jslint node: true */ +"use strict"; + +var util = require("util"); + +var TextProcessors = function() { + this.processors = {}; +}; + +TextProcessors.prototype.registerTextProcessor = function(type,processor) { + this.processors[type] = processor; +}; + +TextProcessors.prototype.parse = function(type,text) { + var processor = this.processors[type]; + if(!processor) { + processor = this.processors["text/x-tiddlywiki"]; + } + if(processor) { + return processor.parse(text); + } else { + return null; + } +}; + +exports.TextProcessors = TextProcessors; diff --git a/js/Tiddler.js b/js/Tiddler.js index 57d144d8a..7492bbcfe 100755 --- a/js/Tiddler.js +++ b/js/Tiddler.js @@ -116,27 +116,4 @@ Tiddler.specialTiddlerFieldParsers = { } }; -Tiddler.prototype.getParseTree = function() { - if(!this.parseTree) { - var type = this.fields.type || "application/x-tiddlywikitext", - parser = Tiddler.tiddlerTextParsers[type]; - if(parser) { - this.parseTree = Tiddler.tiddlerTextParsers[type].call(this); - } - } - return this.parseTree; -}; - -Tiddler.tiddlerTextParsers = { - "application/x-tiddlywikitext": function() { - return new WikiTextParser(this.fields.text); - }, - "application/javascript": function() { - // Would be useful to parse so that we can do syntax highlighting and debugging - }, - "application/json": function() { - return JSON.parse(this.fields.text); - } -}; - exports.Tiddler = Tiddler; diff --git a/js/WikiStore.js b/js/WikiStore.js index d50e3979c..e739503a6 100755 --- a/js/WikiStore.js +++ b/js/WikiStore.js @@ -4,9 +4,13 @@ var Tiddler = require("./Tiddler.js").Tiddler, util = require("util"); -var WikiStore = function WikiStore(shadowStore) { +var WikiStore = function WikiStore(options) { this.tiddlers = {}; - this.shadows = shadowStore === undefined ? new WikiStore(null) : shadowStore; + this.shadows = options.shadowStore !== undefined ? options.shadowStore : new WikiStore({ + shadowStore: null, + textProcessors: options.textProcessors + }); + this.textProcessors = options.textProcessors; }; WikiStore.prototype.clear = function() { @@ -50,4 +54,22 @@ WikiStore.prototype.forEachTiddler = function(callback) { } }; +WikiStore.prototype.parseTiddler = function(title) { + var tiddler = this.getTiddler(title); + if(tiddler) { + return this.textProcessors.parse(tiddler.fields.type,tiddler.fields.text); + } else { + return null; + } +} + +WikiStore.prototype.renderTiddler = function(type,title) { + var parser = this.parseTiddler(title); + if(parser) { + return parser.render(type,parser.children,this,title); + } else { + return null; + } +} + exports.WikiStore = WikiStore; diff --git a/js/WikiTextParser.js b/js/WikiTextParser.js index 7d00846e9..793db9f3b 100644 --- a/js/WikiTextParser.js +++ b/js/WikiTextParser.js @@ -26,17 +26,18 @@ Text nodes are: /*jslint node: true */ "use strict"; -var wikiTextRules = require("./WikiTextRules.js").wikiTextRules, +var WikiTextRenderer = require("./WikiTextRenderer.js").WikiTextRenderer, utils = require("./Utils.js"), util = require("util"); -var WikiTextParser = function(text) { +var WikiTextParser = function(text,processor) { + this.processor = processor; this.autoLinkWikiWords = true; this.source = text; this.nextMatch = 0; - this.tree = []; + this.children = []; this.output = null; - this.subWikify(this.tree); + this.subWikify(this.children); }; WikiTextParser.prototype.outputText = function(place,startPos,endPos) { @@ -58,8 +59,8 @@ WikiTextParser.prototype.subWikifyUnterm = function(output) { var oldOutput = this.output; this.output = output; // Get the first match - wikiTextRules.rulesRegExp.lastIndex = this.nextMatch; - var ruleMatch = wikiTextRules.rulesRegExp.exec(this.source); + this.processor.rulesRegExp.lastIndex = this.nextMatch; + var ruleMatch = this.processor.rulesRegExp.exec(this.source); while(ruleMatch) { // Output any text before the match if(ruleMatch.index > this.nextMatch) @@ -68,18 +69,18 @@ WikiTextParser.prototype.subWikifyUnterm = function(output) { this.matchStart = ruleMatch.index; this.matchLength = ruleMatch[0].length; this.matchText = ruleMatch[0]; - this.nextMatch = wikiTextRules.rulesRegExp.lastIndex; + this.nextMatch = this.processor.rulesRegExp.lastIndex; // Figure out which rule matched and call its handler var t; for(t=1; t` format + +### Tiddler.js + +Tiddlers are an immutable dictionary of name:value pairs called fields. Values can be a string, an array of strings, or a JavaScript date object. + +The only field that is required is the `title` field, but useful tiddlers also have a `text` field, and some or all of the standard fields `modified`, `modifier`, `created`, `creator`, `tags` and `type`. + +Hardcoded in the system is the knowledge that the 'tags' field is a string array, and that the 'modified' and 'created' fields are dates. All other fields are strings. + +#### var tiddler = new Tiddler([srcFields{,srcFields}]) + +Create a Tiddler given a series of sources of fields which can either be a plain hashmap of name:value pairs or an existing tiddler to clone. Fields in later sources overwrite the same field specified in earlier sources. + +The hashmaps can specify the "modified" and "created" fields as strings in YYYYMMDDHHMMSSMMM format or as JavaScript date objects. The "tags" field can be given as a JavaScript array of strings or as a TiddlyWiki quoted string (eg, "one [[two three]]"). + +#### tiddler.fields + +Returns a hashmap of tiddler fields, which can be used for read-only access + +#### tiddler.hasTag(tag) + +Returns a Boolean indicating whether the tiddler has a particular tag. + +#### tiddler.cache(name[,value]) + +Returns or sets the value of a named cache object associated with the tiddler. + +### TiddlerConverters.js + +This class acts as a factory for tiddler serializers and deserializers. + +#### var tiddlerConverters = new TiddlerConverters() + +Creates a tiddler converter factory + +#### tiddlerConverters.registerSerializer(extension,mimeType,serializer) + +Registers a function that knows how to serialise a tiddler into a representation identified by a file extension and a MIME type. The serializer is called with a tiddler to convert and returns the string representation: + + Tiddler.registerSerializer(".tiddler","application/x-tiddler-html-div",function (tiddler) { + return "" + ""; + }); + +#### tiddlerConverters.registerDeserializer(extension,mimeType,deserializer) + +Registers a function that knows how to deserialize one or more tiddlers from a block of text identified by a particular file extension and a MIME type. The deserializer is called with the text to convert and should return an array of tiddler field hashmaps: + + Tiddler.registerDeserializer(".tid","application/x-tiddler",function (text,srcFields) { + var fields = copy(SrcFields); + // Assemble the fields from the text + return [fields]; + }); + +#### tiddlerConverters.deserialize(type,text,srcFields) + +Given a block of text and a MIME type or file extension, returns an array of hashmaps of tiddler fields. One or more source fields can be provided to pre-populate the tiddler before the text is parsed. + +If the type is not recognised then the raw text is assigned to the `text` field. + +#### tiddlerConverters.serialize(tiddler,type) + +Serializes a tiddler into a text representation identified by a MIME type or file extension. + +For example: + + console.log(tiddlerConverters.serialize(tiddler,".tid")); + +### TiddlerInput.js and TiddlerOutput.js + +Contain classes that can be registered with a TiddlerConverters object to common formats. + +#### TiddlerInput.register(tiddlerConverters) + +Registers deserializers for these input types: + + Extension MIME types Description + --------- --------- ----------- + .tiddler application/x-tiddler-html-div TiddlyWiki storeArea-style
+ .tid application/x-tiddler TiddlyWeb-style tiddler text file + .txt text/plain plain text file interpreted as the tiddler text + .html text/html plain HTML file interpreted as the tiddler text + .js application/javascript JavaScript file interpreted as the tiddler text + .json application/json JSON object containing an array of tiddler field hashmaps + .tiddlywiki application/x-tiddlywiki TiddlyWiki HTML document containing one or more tiddler
s + +#### TiddlerOutput.register(tiddlerConverters) + +Registers serializers for these output types: + + Extension MIME types Description + --------- --------- ----------- + .tiddler application/x-tiddler-html-div TiddlyWiki storeArea-style
+ .tid application/x-tiddler TiddlyWeb-style tiddler text file + +### TextProcessors.js + +Text processors are components that know how to parse and render tiddlers of particular types. The core of TiddlyWiki is the WikiText processor, which can parse TiddlyWiki wikitext into a JavaScript object tree representation, and then render the tree into HTML or plain text. Other text processors planned include: + +* `JSONText` to allow JSON objects to display nicely, and make their content available with TiddlyWiki section/slice notation +* `CSSText` to parse CSS, and process extensions such as transclusion, theming and variables +* `JavaScriptText` to parse JavaScript tiddlers for clearer display, and allow sandboxed execution through compilation + +Note that text processors encapsulate two operations: parsing into a tree, and rendering that tree into text representations. Parsing doesn't need a context, but rendering needs to have access to a context consisting of a WikiStore to use to retrieve any referenced tiddlers, and the title of the tiddler that is being rendered. + +#### textProcessors = new TextProcessors() + +Applications should create a TextProcessors object to keep track of the available text processors. + +#### textProcessors.registerTextProcessor(mimeType,textProcessor) + +Registers an instance of a text processor class to handle text with a particular MIME type. For example: + + var options = { + ... + }; + textProcessors.registerTextProcessor("text/x-tiddlywiki",new WikiTextProcessor(options)); + +The text processor object must support the following methods: + + // Parses some text and returns a parse tree object + var parseTree = textProcessor.parse(text) + +Parser objects support the following methods: + + // Renders a subnode of the parse tree to a representation identified by MIME type, + // as if rendered within the context of the specified WikiStore and tiddler title + var renderedText = parseTree.render(type,treenode,store,title) + +#### textProcessors.parse(type,text) + +Chooses a text processor based on the MIME type of the content and calls the `parse` method to parse the text into a parse tree. Returns null if the type was not recognised by a registered parser. + +If the MIME type is unrecognised or unknown, it defaults to "text/x-tiddlywiki". + +### WikiTextProcessor.js + +A text processor that parses and renders TiddlyWiki style wiki text. + +This module privately includes the following modules: + +* WikiTextParser.js containing the wiki text parsing engine +* WikiTextRules.js containing the rules driving the wiki text parsing engine +* WikiTextRenderer.js containing the wiki text rendering engine +* WikiTextMacros.js containing the predefined macros used by the renderer + +#### var wikiTextProcessor = new WikiTextProcessor(options) + +Creates a new instance of the wiki text processor with the specified options. The options are a hashmap of optional members as follows: + +* **enableRules:** An array of names of wiki text rules to enable. If not specified, all rules are available +* **extraRules:** An array of additional rule handlers to add +* **enableMacros:** An array of names of macros to enable. If not specified, all macros are available +* **extraMacros:** An array of additional macro handlers to add + +### WikiStore.js + +A collection of uniquely titled tiddlers. Although the tiddlers themselves are immutable, new tiddlers can be stored under an existing title, replacing the previous tiddler. + +Each wiki store is connected to a shadow store that is also a WikiStore() object. Under certain circumstances, when an attempt is made to retrieve a tiddler that doesn't exist in the store, the search continues into its shadow store (and so on, if the shadow store itself has a shadow store). + +#### var store = new WikiStore(options) + +Creates a new wiki store. The options are a hashmap of optional members as follows: + +* **textProcessors:** A reference to the TextProcessors() instance to be used to resolve parsing and rendering requests +* **shadowStore:** An optional reference to an existing WikiStore to use as the source of shadow tiddlers. Pass null to disable shadow tiddlers for the new store + +#### store.shadows + +Exposes a reference to the shadow store for this store. + +#### store.clear() + +Clears the store of all tiddlers. + +#### store.getTiddler(title) + +Attempts to retrieve the tiddler with a given title. Returns `null` if the tiddler doesn't exist. + +#### store.getTiddlerText(title,defaultText) + +Retrieves the text of a particular tiddler. If the tiddler doesn't exist, then the defaultText is returned, or `null` if not specified. + +#### store.deleteTiddler(title) + +Deletes the specified tiddler from the store. + +#### store.tiddlerExists(title) + +Returns a boolean indicating whether a particular tiddler exists. + +#### store.addTiddler(tiddler) + +Adds the specified tiddler object to the store. The tiddler can be specified as a Tiddler() object or a hashmap of tiddler fields. + +#### store.forEachTiddler([sortField,]callback) + +Invokes a callback for each tiddler in the store, optionally sorted by a particular field. The callback is called with the title of the tiddler and a reference to the tiddler itself. For example: + + store.forEachTiddler(function(title,tiddler) { + console.log(title); + }); + +#### store.parseTiddler(title) + +Returns the parse tree object for a tiddler, which may be cached within the tiddler. + +#### store.renderTiddler(type,title) + +Returns a dynamically generated rendering of the tiddler in a representation identified by a MIME type. + +### Recipe.js + +The Recipe() class loads a TiddlyWiki recipe file, resolving references to subrecipe files. Tiddlers referenced by the recipe are loaded into a WikiStore. A fully loaded recipe can then be cooked to produce an HTML or RSS TiddlyWiki representation of the recipe. + +#### var recipe = new Recipe(options,callback) + +Creates a new Recipe object by loading the specified recipe file. On completion the callback is invoked with a single parameter `err` that is null if the recipe loading was successful, or an Error() object otherwise. + + var recipe = new Recipe({ + filepath: "recent.recipe", + tiddlerConverters: tiddlerConverters, + store: store + },function callback(err) { + if(err) { + throw err; + } else { + console.log(recipe.cook()) + } + } + +Options is a hashmap with the following mandatory fields: + +* **filepath:** The filepath to the recipe file to load +* **tiddlerConverters:** The TiddlerConverters() object to use to serialize and deserialize tiddlers +* **textProcessors:** The TextProcessors() object to use to parse and render tiddler text +* **store:** The WikiStore object to use to store the tiddlers in the recipe + +The options can also contain these optional fields: + +* (none at present) + +#### recipe.cook() + +Cooks a TiddlyWiki HTML file from the recipe and returns it as a string. + +#### recipe.cookRss() + +Cooks a TiddlyWiki RSS file from the recipe and returns it as a string. diff --git a/tiddlywiki.js b/tiddlywiki.js index b978121d9..4da1658f7 100644 --- a/tiddlywiki.js +++ b/tiddlywiki.js @@ -8,9 +8,10 @@ TiddlyWiki command line interface var WikiStore = require("./js/WikiStore.js").WikiStore, Tiddler = require("./js/Tiddler.js").Tiddler, Recipe = require("./js/Recipe.js").Recipe, - Tiddler = require("./js/Tiddler.js").Tiddler, tiddlerInput = require("./js/TiddlerInput.js"), tiddlerOutput = require("./js/TiddlerOutput.js"), + TextProcessors = require("./js/TextProcessors.js").TextProcessors, + WikiTextProcessor = require("./js/WikiTextProcessor.js").WikiTextProcessor, TiddlerConverters = require("./js/TiddlerConverters.js").TiddlerConverters, util = require("util"), fs = require("fs"), @@ -42,19 +43,27 @@ var parseOptions = function(args,defaultSwitch) { return result; }; -var tiddlerConverters = new TiddlerConverters(), +var textProcessors = new TextProcessors(), + tiddlerConverters = new TiddlerConverters(), switches = parseOptions(Array.prototype.slice.call(process.argv,2),"dummy"), - store = new WikiStore(), + store = new WikiStore({ + textProcessors: textProcessors + }), recipe = null, lastRecipeFilepath = null, currSwitch = 0; + +textProcessors.registerTextProcessor("text/x-tiddlywiki",new WikiTextProcessor({})); // Register the standard tiddler serializers and deserializers tiddlerInput.register(tiddlerConverters); tiddlerOutput.register(tiddlerConverters); // Add the shadow tiddlers that are built into TiddlyWiki -var shadowShadowStore = new WikiStore(null), +var shadowShadowStore = new WikiStore({ + textProcessors: textProcessors, + shadowStore: null + }), shadowShadows = [ {title: "StyleSheet", text: ""}, {title: "MarkupPreHead", text: ""}, @@ -123,7 +132,8 @@ var commandLineSwitches = { recipe = new Recipe({ filepath: args[0], store: store, - tiddlerConverters: tiddlerConverters + tiddlerConverters: tiddlerConverters, + textProcessors: textProcessors },function() { callback(null); }); @@ -203,7 +213,9 @@ var commandLineSwitches = { // Dumbly, this implementation wastes the recipe processing that happened on the --recipe switch http.createServer(function(request, response) { response.writeHead(200, {"Content-Type": "text/html"}); - store = new WikiStore(); + store = new WikiStore({ + textProcessors: textProcessors + }); recipe = new Recipe(store,lastRecipeFilepath,function() { response.end(recipe.cook(), "utf8"); }); @@ -219,7 +231,7 @@ var commandLineSwitches = { tiddler = store.getTiddler(title); if(tiddler) { response.writeHead(200, {"Content-Type": "text/html"}); - response.end(tiddler.getParseTree().render("text/html"),"utf8"); + response.end(store.renderTiddler("text/html",title),"utf8"); } else { response.writeHead(404); response.end(); diff --git a/wikitest.js b/wikitest.js index 4d64a0cba..3c405b274 100644 --- a/wikitest.js +++ b/wikitest.js @@ -14,7 +14,8 @@ verifying that the output matches `.html` and `.txt`. var Tiddler = require("./js/Tiddler.js").Tiddler, WikiStore = require("./js/WikiStore.js").WikiStore, - WikiTextRenderer = require("./js/WikiTextRenderer.js").WikiTextRenderer, + TextProcessors = require("./js/TextProcessors.js").TextProcessors, + WikiTextProcessor = require("./js/WikiTextProcessor.js").WikiTextProcessor, TiddlerConverters = require("./js/TiddlerConverters.js").TiddlerConverters, tiddlerInput = require("./js/TiddlerInput.js"), utils = require("./js/Utils.js"), @@ -23,12 +24,16 @@ var Tiddler = require("./js/Tiddler.js").Tiddler, path = require("path"); var testdirectory = process.argv[2], + textProcessors = new TextProcessors(), tiddlerConverters = new TiddlerConverters(), - store = new WikiStore(), + store = new WikiStore({ + textProcessors: textProcessors + }), files = fs.readdirSync(testdirectory), titles = [], f,t,extname,basename; +textProcessors.registerTextProcessor("text/x-tiddlywiki",new WikiTextProcessor({})); tiddlerInput.register(tiddlerConverters); for(f=0; f