Memory efficient linked list (#5380)

* Outlines of the mem efficient linked list

Need to stop for now. Found problem with $tw.utils.pushTop that I need
consultation for.

* Link list throws when given non-string vals

* Think I got rid of the last LinkList infinite loops

* LinkedList push better; fixed coding conventions

* Cleaning up LinkedList code and tests

* Ready to ship new mem efficient Linked List

* Switching to double quotes in LinkedList

core/modules/utils/linked-list.js

@@ -14,105 +14,181 @@ function LinkedList() {
 };
 
 LinkedList.prototype.clear = function() {
-	this.index = Object.create(null);
 	// LinkedList performs the duty of both the head and tail node
-	this.next = this;
+	this.next = Object.create(null);
-	this.prev = this;
+	this.prev = Object.create(null);
+	this.first = undefined;
+	this.last = undefined;
 	this.length = 0;
 };
 
 LinkedList.prototype.remove = function(value) {
 	if($tw.utils.isArray(value)) {
+		for(var t=0; t<value.length; t++) {
+			_assertString(value[t]);
+		}
 		for(var t=0; t<value.length; t++) {
 			_removeOne(this,value[t]);
 		}
 	} else {
+		_assertString(value);
 		_removeOne(this,value);
 	}
 };
 
+/*
+Push behaves like array.push and accepts multiple string arguments. But it also
+accepts a single array argument too, to be consistent with its other methods.
+*/
 LinkedList.prototype.push = function(/* values */) {
-	for(var i = 0; i < arguments.length; i++) {
+	var values = arguments;
-		var value = arguments[i];
+	if($tw.utils.isArray(values[0])) {
-		var node = {value: value};
+		values = values[0];
-		var preexistingNode = this.index[value];
-		_linkToEnd(this,node);
-		if(preexistingNode) {
-			// We want to keep pointing to the first instance, but we want
-			// to have that instance (or chain of instances) point to the
-			// new one.
-			while (preexistingNode.copy) {
-				preexistingNode = preexistingNode.copy;
-			}
-			preexistingNode.copy = node;
-		} else {
-			this.index[value] = node;
-		}
 	}
+	for(var i = 0; i < values.length; i++) {
+		_assertString(values[i]);
+	}
+	for(var i = 0; i < values.length; i++) {
+		_linkToEnd(this,values[i]);
+	}
+	return this.length;
 };
 
 LinkedList.prototype.pushTop = function(value) {
 	if($tw.utils.isArray(value)) {
+		for (var t=0; t<value.length; t++) {
+			_assertString(value[t]);
+		}
 		for(var t=0; t<value.length; t++) {
 			_removeOne(this,value[t]);
 		}
-		this.push.apply(this,value);
+		for(var t=0; t<value.length; t++) {
-	} else {
+			_linkToEnd(this,value[t]);
-		var node = _removeOne(this,value);
-		if(!node) {
-			node = {value: value};
-			this.index[value] = node;
-		} else {
-			// Put this node at the end of the copy chain.
-			var preexistingNode = node;
-			while(preexistingNode.copy) {
-				preexistingNode = preexistingNode.copy;
-			}
-			// The order of these three statements is important,
-			// because sometimes preexistingNode == node.
-			preexistingNode.copy = node;
-			this.index[value] = node.copy;
-			node.copy = undefined;
 		}
-		_linkToEnd(this,node);
+	} else {
+		_assertString(value);
+		_removeOne(this,value);
+		_linkToEnd(this,value);
 	}
 };
 
 LinkedList.prototype.each = function(callback) {
-	for(var ptr = this.next; ptr !== this; ptr = ptr.next) {
+	var visits = Object.create(null),
-		callback(ptr.value);
+		value = this.first;
+	while(value !== undefined) {
+		callback(value);
+		var next = this.next[value];
+		if(typeof next === "object") {
+			var i = visits[value] || 0;
+			visits[value] = i+1;
+			value = next[i];
+		} else {
+			value = next;
+		}
 	}
 };
 
 LinkedList.prototype.toArray = function() {
 	var output = [];
-	for(var ptr = this.next; ptr !== this; ptr = ptr.next) {
+	this.each(function(value) { output.push(value); });
-		output.push(ptr.value);
-	}
 	return output;
 };
 
 function _removeOne(list,value) {
-	var node = list.index[value];
+	var prevEntry = list.prev[value],
-	if(node) {
+		nextEntry = list.next[value],
-		node.prev.next = node.next;
+		prev = prevEntry,
-		node.next.prev = node.prev;
+		next = nextEntry;
-		list.length -= 1;
+	if(typeof nextEntry === "object") {
-		// Point index to the next instance of the same value, maybe nothing.
+		next = nextEntry[0];
-		list.index[value] = node.copy;
+		prev = prevEntry[0];
 	}
-	return node;
+	// Relink preceding element.
+	if(list.first === value) {
+		list.first = next
+	} else if(prev !== undefined) {
+		if(typeof list.next[prev] === "object") {
+			if(next === undefined) {
+				// Must have been last, and 'i' would be last element.
+				list.next[prev].pop();
+			} else {
+				var i = list.next[prev].indexOf(value);
+				list.next[prev][i] = next;
+			}
+		} else {
+			list.next[prev] = next;
+		}
+	} else {
+		return;
+	}
+	// Now relink following element
+	// Check "next !== undefined" rather than "list.last === value" because
+	// we need to know if the FIRST value is the last in the list, not the last.
+	if(next !== undefined) {
+		if(typeof list.prev[next] === "object") {
+			if(prev === undefined) {
+				// Must have been first, and 'i' would be 0.
+				list.prev[next].shift();
+			} else {
+				var i = list.prev[next].indexOf(value);
+				list.prev[next][i] = prev;
+			}
+		} else {
+			list.prev[next] = prev;
+		}
+	} else {
+		list.last = prev;
+	}
+	// Delink actual value. If it uses arrays, just remove first entries.
+	if(typeof nextEntry === "object") {
+		nextEntry.shift();
+		prevEntry.shift();
+	} else {
+		list.next[value] = undefined;
+		list.prev[value] = undefined;
+	}
+	list.length -= 1;
 };
 
-function _linkToEnd(list,node) {
+// Sticks the given node onto the end of the list.
-	// Sticks the given node onto the end of the list.
+function _linkToEnd(list,value) {
-	list.prev.next = node;
+	if(list.first === undefined) {
-	node.prev = list.prev;
+		list.first = value;
-	list.prev = node;
+	} else {
-	node.next = list;
+		// Does it already exists?
+		if(list.first === value || list.prev[value] !== undefined) {
+			if(typeof list.next[value] === "string") {
+				list.next[value] = [list.next[value]];
+				list.prev[value] = [list.prev[value]];
+			} else if(typeof list.next[value] === "undefined") {
+				// list.next[value] must be undefined.
+				// Special case. List already has 1 value. It's at the end.
+				list.next[value] = [];
+				list.prev[value] = [list.prev[value]];
+			}
+			list.prev[value].push(list.last);
+			// We do NOT append a new value onto "next" list. Iteration will
+			// figure out it must point to End-of-List on its own.
+		} else {
+			list.prev[value] = list.last;
+		}
+		// Make the old last point to this new one.
+		if(typeof list.next[list.last] === "object") {
+			list.next[list.last].push(value);
+		} else {
+			list.next[list.last] = value;
+		}
+	}
+	list.last = value;
 	list.length += 1;
 };
 
+function _assertString(value) {
+	if(typeof value !== "string") {
+		throw "Linked List only accepts string values, not " + value;
+	}
+};
+
 exports.LinkedList = LinkedList;
 
 })();

editions/test/tiddlers/tests/test-linked-list.js

@@ -8,10 +8,18 @@ Tests the utils.LinkedList class.
 LinkedList was built to behave exactly as $tw.utils.pushTop and
 Array.prototype.push would behave with an array.
 
-Many of these tests function by performing operations on a LinkedList while
+Many of these tests function by performing operations on a paired set of
-performing the equivalent actions on an array with the old utility methods.
+an array and LinkedList. It uses equivalent actions on both.
 Then we confirm that the two come out functionally identical.
 
+NOTE TO FURTHER LINKED LIST DEVELOPERS:
+
+	If you want to add new functionality, like 'shift' or 'unshift', you'll
+	probably need to deal with the fact that Linked List will insert undefined
+	as a first entry into an item's 'prev' array when it's at the front of
+	the list, but it doesn't do the same for the 'next' array when it's at
+	the end. I think you'll probably be better off preventing 'prev' from ever
+	adding undefined.
 \*/
 (function(){
 
@@ -21,127 +29,230 @@ Then we confirm that the two come out functionally identical.
 
 describe("LinkedList class tests", function() {
 
+	// creates and initializes a new {array, list} pair for testing
+	function newPair(initialArray) {
+		var pair = {array: [], list: new $tw.utils.LinkedList()};
+		if (initialArray) {
+			push(pair, initialArray);
+		}
+		return pair;
+	};
+
 	// pushTops a value or array of values into both the array and linked list.
-	function pushTop(array, linkedList, valueOrValues) {
+	function pushTop(pair, valueOrValues) {
-		$tw.utils.pushTop(array, valueOrValues);
+		pair.list.pushTop(valueOrValues);
-		linkedList.pushTop(valueOrValues);
+		$tw.utils.pushTop(pair.array, valueOrValues);
+		return pair;
 	};
 
 	// pushes values into both the array and the linked list.
-	function push(array, linkedList/*, other values */) {
+	function push(pair, values) {
-		var values = Array.prototype.slice(arguments, 2);
+		pair.list.push(values);
-		array.push.apply(array, values);
+		pair.array.push.apply(pair.array, values);
-		linkedList.push.apply(linkedList, values);
+		return pair;
 	};
 
 	// operates a remove action on an array and a linked list in parallel.
-	function remove(array, linkedList, valueOrValues) {
+	function remove(pair, valueOrValues) {
-		$tw.utils.removeArrayEntries(array, valueOrValues);
+		pair.list.remove(valueOrValues);
-		linkedList.remove(valueOrValues);
+		$tw.utils.removeArrayEntries(pair.array, valueOrValues);
+		return pair;
 	};
 
 	// compares an array and a linked list to make sure they match up
-	function compare(array, linkedList) {
+	function compare(pair) {
-		expect(linkedList.toArray()).toEqual(array);
+		expect(pair.list.toArray()).toEqual(pair.array);
-		expect(linkedList.length).toBe(array.length);
+		expect(pair.list.length).toBe(pair.array.length);
+		return pair;
 	};
 
 	it("can pushTop", function() {
-		var array = [];
+		var pair = newPair(["A", "B", "C"]);
-		var list = new $tw.utils.LinkedList();
-		push(array, list, 'A', 'B', 'C');
 		// singles
-		pushTop(array, list, 'X');
+		pushTop(pair, "X");
-		pushTop(array, list, 'B');
+		pushTop(pair, "B");
-		compare(array, list); // A C X B
+		compare(pair); // ACXB
 		//arrays
-		pushTop(array, list, ['X', 'A', 'G', 'A']);
+		pushTop(pair, ["X", "A", "G", "A"]);
 		// If the pushedTopped list has duplicates, they go in unempeded.
-		compare(array, list); // C B X A G A
+		compare(pair); // CBXAGA
 	});
 
 	it("can pushTop with tricky duplicates", function() {
-		var array = [];
+		var pair = newPair(["A", "B", "A", "C", "A", "e"]);
-		var list = new $tw.utils.LinkedList();
-		push(array, list, 'A', 'B', 'A', 'C', 'A', 'end');
 		// If the original list contains duplicates, only one instance is cut
-		pushTop(array, list, 'A');
+		compare(pushTop(pair, "A")); // BACAeA
-		compare(array, list); // B A C A end A
 
 		// And the Llist properly knows the next 'A' to cut if pushed again
-		pushTop(array, list, ['X', 'A']);
+		compare(pushTop(pair, ["X", "A"])); // BCAeAXA
-		compare(array, list); // B C A end A X A
 
 		// One last time, to make sure we maintain the linked chain of copies
-		pushTop(array, list, 'A');
+		compare(pushTop(pair, "A")); // BCeAXAA
-		compare(array, list); // B C end A X A A
+	});
+
+	it("can pushTop a single-value list with itself", function() {
+		// This simple case actually requires special handling in LinkedList.
+		compare(pushTop(newPair(["A"]), "A")); // A
+	});
+
+	it("can remove all instances of a multi-instance value", function() {
+		var pair = compare(remove(newPair(["A", "A"]), ["A", "A"])); //
+		// Now add 'A' back in, since internally it might be using arrays,
+		// even though those arrays must be empty.
+		compare(pushTop(pair, "A")); // A
+		// Same idea, but push something else before readding 'A'
+		compare(pushTop(remove(newPair(["A", "A"]), ["A", "A"]), ["B", "A"])); // BA
+
+		// Again, but this time with other values mixed in
+		compare(remove(newPair(["B", "A", "A", "C"]), ["A", "A"])) // BC;
+		// And again, but this time with value inbetween too.
+		compare(remove(newPair(["B", "A", "X", "Y", "Z", "A", "C"]), ["A", "A"])); // BXYZC
+
+		// One last test, where removing a pair from the end could corrupt
+		// list.last.
+		pair = remove(newPair(["D", "C", "A", "A"]), ["A", "A"]);
+		// But I can't figure out another way to test this. It's wrong
+		// for list.last to be anything other than a string, but I
+		// can't figure out how to make that corruption manifest a problem.
+		// So I dig into its private members. Bleh...
+		expect(typeof pair.list.last).toBe("string");
+	});
+
+	it("can pushTop value linked to by a repeat item", function() {
+		var pair = newPair(["A", "B", "A", "C", "A", "C", "D"]);
+		// This is tricky because that 'C' is referenced by a second 'A'
+		// It WAS a crash before
+		pushTop(pair, "C");
+		compare(pair); // ABAACDC
+	});
+
+	it("can pushTop last value after pair", function() {
+		// The 'next' ptrs for A would be polluted with an extraneous
+		// undefined after the pop, which would make pushing the 'X'
+		// back on problematic.
+		compare(pushTop(newPair(["A", "A", "X"]), "X")); // AACX
+		// And lets try a few other manipulations around pairs
+		compare(pushTop(newPair(["A", "A", "X", "C"]), "X")); // AACX
+		compare(pushTop(newPair(["X", "A", "A"]), "X")); // AAX
+		compare(pushTop(newPair(["C", "X", "A", "A"]), "X")); // CAAX
 	});
 
 	it("can handle particularly nasty pushTop pitfall", function() {
-		var array = [];
+		var pair = newPair(["A", "B", "A", "C"]);
-		var list = new $tw.utils.LinkedList();
+		pushTop(pair, "A"); // BACA
-		push(array, list, 'A', 'B', 'A', 'C');
+		pushTop(pair, "X"); // BACAX
-		pushTop(array, list, 'A'); // BACA
+		remove(pair, "A");  // BCAX
-		pushTop(array, list, 'X'); // BACAX
+		pushTop(pair, "A"); // BCXA
-		remove(array, list, 'A');  // BCAX
+		remove(pair, "A");  // BCX
-		pushTop(array, list, 'A'); // BCXA
-		remove(array, list, 'A');  // BCX
 
 		// But! The way I initially coded the copy chains, a mystery A could
 		// hang around.
-		compare(array, list); // B C X
+		compare(pair); // BCX
+	});
+
+	it("can handle past-duplicate items when pushing", function() {
+		var pair = newPair(["X", "Y", "A", "C", "A"]);
+		// Removing an item, when it has a duplicat at the list's end
+		remove(pair, "A");
+		compare(pair); // XYCA
+		// This actually caused an infinite loop once. So important test here.
+		push(pair, ["A"]);
+		compare(pair); // XYCAA
+		pushTop(pair, "A") // switch those last As
+		compare(pair); // XYCAA
+		remove(pair, ["A", "A"]); // Remove all As, then add them back
+		pushTop(pair, ["A", "A"])
+		compare(pair); // XYCAA
 	});
 
 	it("can push", function() {
-		var array = [];
 		var list = new $tw.utils.LinkedList();
-		push(array, list, 'A', 'B', 'C');
 		// singles
-		push(array, list, 'B');
+		expect(list.push("A")).toBe(1);
-		compare(array, list); // A B C B
+		expect(list.push("B")).toBe(2);
-
 		// multiple args
-		push(array, list, 'A', 'B', 'C');
+		expect(list.push("C", "D", "E")).toBe(5);
-		compare(array, list); // A B C B A B C
+		// array arg allowed
+		expect(list.push(["F", "G"])).toBe(7);
+		// No-op
+		expect(list.push()).toBe(7);
+		expect(list.toArray()).toEqual(["A", "B", "C", "D", "E", "F", "G"]);
+	});
+
+	it("can handle empty string", function() {
+		compare(newPair(["", "", ""])); // ___
+		compare(push(newPair([""]), [""])); // __
+		compare(pushTop(newPair(["", "", ""]), ["A", ""])); // __A_
+		compare(remove(newPair(["", "A"]), "A")); // _
+		compare(push(newPair(["", "A"]), ["A"])); // _AA
+		compare(remove(newPair(["A", ""]), "A")); // _
+		compare(push(newPair(["A", ""]), ["A"])); // A_A
+
+		// This one is tricky but precise. Remove 'B', and 'A' might mistake
+		// it as being first in the list since it's before ''. 'C' would get
+		// blasted from A's prev reference array.
+		compare(remove(newPair(["C", "A", "", "B", "A"]), ["B", "A"])); // C_A
+		// Same idea, but with A mistaking B for being at the list's end, and
+		// thus removing C from its 'next' reference array.
+		compare(remove(newPair(["A", "B", "", "A", "C"]), ["B", "A"])); // _AC
+	});
+
+	it("will throw if told to push non-strings", function() {
+		var message = "Linked List only accepts string values, not ";
+		var list = new $tw.utils.LinkedList();
+		expect(() => list.push(undefined)).toThrow(message + "undefined");
+		expect(() => list.pushTop(undefined)).toThrow(message + "undefined");
+		expect(() => list.pushTop(["c", undefined])).toThrow(message + "undefined");
+		expect(() => list.pushTop(5)).toThrow(message + "5");
+		expect(() => list.pushTop(null)).toThrow(message + "null");
+
+		// now lets do a quick test to make sure this exception
+		// doesn't leave any side-effects
+		// A.K.A Strong guarantee
+		var pair = newPair(["A", "5", "B", "C"]);
+		expect(() => pushTop(pair, 5)).toThrow(message + "5");
+		compare(pair);
+		expect(() => push(pair, ["D", 7])).toThrow(message + "7");
+		compare(pair);
+		expect(() => remove(pair, 5)).toThrow(message + "5");
+		compare(pair);
+		// This is the tricky one. 'A' and 'B' should not be removed or pushed.
+		expect(() => pushTop(pair, ["A", "B", null])).toThrow(message + "null");
+		compare(pair);
+		expect(() => remove(pair, ["A", "B", null])).toThrow(message + "null");
+		compare(pair);
 	});
 
 	it("can clear", function() {
 		var list = new $tw.utils.LinkedList();
-		list.push('A', 'B', 'C');
+		list.push("A", "B", "C");
 		list.clear();
 		expect(list.toArray()).toEqual([]);
 		expect(list.length).toBe(0);
 	});
 
 	it("can remove", function() {
-		var array = [];
 		var list = new $tw.utils.LinkedList();
-		push(array, list, 'A', 'x', 'C', 'x', 'D', 'x', 'E', 'x');
+		list.push(["A", "x", "C", "x", "D", "x", "E", "x"]);
 		// single
-		remove(array, list, 'x');
+		list.remove("x");
-		compare(array, list); // A C x D x E x
-
 		// arrays
-		remove(array, list, ['x', 'A', 'x']);
+		list.remove(["x", "A", "XXX", "x"]);
-		compare(array, list); // C D E x
+		expect(list.toArray()).toEqual(["C", "D", "E", "x"]);
 	});
 
-	it('can ignore removal of nonexistent items', function() {
+	it("can ignore removal of nonexistent items", function() {
-		var array = [];
+		var pair = newPair(["A", "B", "C", "D"]);
-		var list = new $tw.utils.LinkedList();
-		push(array, list, 'A', 'B', 'C', 'D');
 		// single
-		remove(array, list, 'Z');
+		compare(remove(pair, "Z")); // ABCD
-		compare(array, list); // A B C D
 
 		// array
-		remove(array, list, ['Z', 'B', 'X']);
+		compare(remove(pair, ["Z", "B", "X"])); // ACD
-		compare(array, list); // A C D
 	});
 
-	it('can iterate with each', function() {
+	it("can iterate with each", function() {
 		var list = new $tw.utils.LinkedList();
-		list.push('0', '1', '2', '3');
+		list.push("0", "1", "2", "3");
 		var counter = 0;
 		list.each(function(value) {
 			expect(value).toBe(counter.toString());
@@ -149,6 +260,11 @@ describe("LinkedList class tests", function() {
 		});
 		expect(counter).toBe(4);
 	});
+
+	it("can iterate a list of the same item", function() {
+		// Seems simple. Caused an infinite loop during development.
+		compare(newPair(["A", "A"]));
+	});
 });
 
 })();