CSV parser improvements (#7042)

2026-09-19 00:00:43 +00:00 · 2022-11-20 17:51:01 +00:00
parent b8a30091ee
commit 4f7b10e055
5 changed files with 446 additions and 40 deletions
@@ -13,6 +13,11 @@ The CSV text parser processes CSV files into a table wrapped in a scrollable wid
 "use strict";

 var CsvParser = function(type,text,options) {
+	// Special handler for tab-delimited files
+	if (type === 'text/tab-delimited-values' && !options.separator) {
+		options.separator = "\t";
+	}
+	
 	// Table framework
 	this.tree = [{
 		"type": "scrollable", "children": [{
@@ -24,30 +29,33 @@ var CsvParser = function(type,text,options) {
 		}]
 	}];
 	// Split the text into lines
-	var lines = text.split(/\r?\n/mg),
+	var lines = $tw.utils.parseCsvString(text, options),
 		tag = "th";
+	var maxColumns = 0;
+	$tw.utils.each(lines, function(columns) {
+		maxColumns = Math.max(columns.length, maxColumns);
+	});
+	
 	for(var line=0; line<lines.length; line++) {
-		var lineText = lines[line];
-		if(lineText) {
-			var row = {
-					"type": "element", "tag": "tr", "children": []
-				};
-			var columns = lineText.split(",");
-			for(var column=0; column<columns.length; column++) {
-				row.children.push({
-						"type": "element", "tag": tag, "children": [{
-							"type": "text",
-							"text": columns[column]
-						}]
-					});
-			}
-			tag = "td";
-			this.tree[0].children[0].children[0].children.push(row);
+		var columns = lines[line];
+		var row = {
+			"type": "element", "tag": "tr", "children": []
+		};
+		for(var column=0; column<maxColumns; column++) {
+			row.children.push({
+				"type": "element", "tag": tag, "children": [{
+					"type": "text",
+					"text": columns[column] || ''
+				}]
+			});
 		}
+		tag = "td";
+		this.tree[0].children[0].children[0].children.push(row);
 	}
 };

 exports["text/csv"] = CsvParser;
+exports["text/tab-delimited-values"] = CsvParser;

 })();

@@ -12,35 +12,113 @@ A barebones CSV parser
 /*global $tw: false */
 "use strict";

+var QUOTE = '"';
+
+var getCellInfo = function(text, start, length, SEPARATOR) {
+	var isCellQuoted = text.charAt(start) === QUOTE;
+	var cellStart = isCellQuoted ? start + 1 : start;
+	
+	if (text.charAt(i) === SEPARATOR) {
+		return [cellStart, cellStart, false];
+	}
+	
+	for (var i = cellStart; i < length; i++) {
+		var cellCharacter = text.charAt(i);
+		var isEOL = cellCharacter === "\n" || cellCharacter === "\r";
+		
+		if (isEOL && !isCellQuoted) {
+			return [cellStart, i, false];
+			
+		} else if (cellCharacter === SEPARATOR && !isCellQuoted) {
+			return [cellStart, i, false];
+			
+		} else if (cellCharacter === QUOTE && isCellQuoted) {
+			var nextCharacter = i + 1 < length ? text.charAt(i + 1) : '';
+			if (nextCharacter !== QUOTE) {
+				return [cellStart, i, true];
+			} else {
+				i++;
+			}
+		}
+	}
+	
+	return [cellStart, i, isCellQuoted];
+}
+	
+exports.parseCsvString = function(text, options) {
+	if (!text) {
+		return [];
+	}
+	
+	options = options || {};
+	var SEPARATOR = options.separator || ",",
+		length = text.length,
+		rows = [],
+		nextRow = [];
+		
+	for (var i = 0; i < length; i++) {
+		var cellInfo = getCellInfo(text, i, length, SEPARATOR);
+		var cellText = text.substring(cellInfo[0], cellInfo[1]);
+		if (cellInfo[2]) {
+			cellText = cellText.replace(/""/g, '"');
+			cellInfo[1]++;
+		}
+		nextRow.push(cellText);
+		
+		i = cellInfo[1];
+		
+		var character = text.charAt(i);
+		var nextCharacter = i + 1 < length ? text.charAt(i + 1) : '';
+		
+		if (character === "\r" || character === "\n") {
+			// Edge case for empty rows
+			if (nextRow.length === 1 && nextRow[0] === '') {
+				nextRow.length = 0;
+			}
+			rows.push(nextRow);
+			nextRow = [];
+			
+			if (character === "\r") {
+				var nextCharacter = i + 1 < length ? text.charAt(i + 1) : '';
+				
+				if (nextCharacter === "\n") {
+					i++;
+				}
+			}
+		}
+	}
+	
+	// Special case if last cell in last row is an empty cell
+	if (text.charAt(length - 1) === SEPARATOR) {
+		nextRow.push("");
+	}
+	
+	rows.push(nextRow);
+	
+	return rows;
+}
+
 /*
 Parse a CSV string with a header row and return an array of hashmaps.
 */
 exports.parseCsvStringWithHeader = function(text,options) {
-	options = options || {};
-	var separator = options.separator || ",",
-		rows = text.split(/\r?\n/mg).map(function(row) {
-			return $tw.utils.trim(row);
-		}).filter(function(row) {
-			return row !== "";
-		});
-	if(rows.length < 1) {
-		return "Missing header row";
-	}
-	var headings = rows[0].split(separator),
-		results = [];
-	for(var row=1; row<rows.length; row++) {
-		var columns = rows[row].split(separator),
-			columnResult = Object.create(null);
-		if(columns.length !== headings.length) {
-			return "Malformed CSV row '" + rows[row] + "'";
+	var csv = $tw.utils.parseCsvString(text, options);
+	var headers = csv[0];
+	
+	csv = csv.slice(1);
+	for (var i = 0; i < csv.length; i++) {
+		var row = csv[i];
+		var rowObject = Object.create(null);
+
+		for(var columnIndex=0; columnIndex<headers.length; columnIndex++) {
+			var columnName = headers[columnIndex];
+			if (columnName) {
+				rowObject[columnName] = $tw.utils.trim(row[columnIndex] || "");
+			}
 		}
-		for(var column=0; column<columns.length; column++) {
-			var columnName = headings[column];
-			columnResult[columnName] = $tw.utils.trim(columns[column] || "");
-		}
-		results.push(columnResult);
+		csv[i] = rowObject;
 	}
-	return results;
+	return csv;
 }

 })();