From a292d33830d3fbd2ef9a9331b85436eabcf5c8d2 Mon Sep 17 00:00:00 2001 From: Jonathan Coates Date: Wed, 25 Jun 2025 22:49:23 +0100 Subject: [PATCH] Syntax highlighting for multiline tokens in edit I don't love the implementation of this (see discussion in #2220), but it's better than nothing. Wow, the editor really needs a bit of a rewrite, the code is kinda messy. Fixes #1396. --- .../modules/main/cc/internal/syntax/lexer.lua | 134 ++++++++++----- .../computercraft/lua/rom/programs/edit.lua | 153 +++++++++++------- 2 files changed, 191 insertions(+), 96 deletions(-) diff --git a/projects/core/src/main/resources/data/computercraft/lua/rom/modules/main/cc/internal/syntax/lexer.lua b/projects/core/src/main/resources/data/computercraft/lua/rom/modules/main/cc/internal/syntax/lexer.lua index 2d8d91938..e4b5110fb 100644 --- a/projects/core/src/main/resources/data/computercraft/lua/rom/modules/main/cc/internal/syntax/lexer.lua +++ b/projects/core/src/main/resources/data/computercraft/lua/rom/modules/main/cc/internal/syntax/lexer.lua @@ -101,16 +101,21 @@ local function lex_number(context, str, start) return tokens.NUMBER, pos - 1 end ---- Lex a quoted string. --- --- @param context The current parser context. --- @tparam string str The string we're lexing. --- @tparam number start_pos The start position of the string. --- @tparam string quote The quote character, either " or '. --- @treturn number The token id for strings. --- @treturn number The new position. -local function lex_string(context, str, start_pos, quote) - local pos = start_pos + 1 +local lex_string_zap + +--[[- Lex a quoted string. + +@param context The current parser context. +@tparam string str The string we're lexing. +@tparam number pos The position to start lexing from. +@tparam number start_pos The actual start position of the string. +@tparam string quote The quote character, either " or '. +@treturn number The token id for strings. +@treturn number The new position. +@treturn nil A placeholder value. +@treturn table|nil The continuation function when the string is not finished. +]] +local function lex_string(context, str, pos, start_pos, quote) while true do local c = sub(str, pos, pos) if c == quote then @@ -125,24 +130,9 @@ local function lex_string(context, str, start_pos, quote) pos = newline(context, str, pos + 1, c) elseif c == "" then context.report(errors.unfinished_string_escape, start_pos, pos, quote) - return tokens.STRING, pos + return tokens.STRING, pos, nil, { lex_string, 1, 1, quote } elseif c == "z" then - pos = pos + 2 - while true do - local next_pos, _, c = find(str, "([%S\r\n])", pos) - - if not next_pos then - context.report(errors.unfinished_string, start_pos, #str, quote) - return tokens.STRING, #str - end - - if c == "\n" or c == "\r" then - pos = newline(context, str, next_pos, c) - else - pos = next_pos - break - end - end + return lex_string_zap(context, str, pos + 2, start_pos, quote) else pos = pos + 2 end @@ -152,6 +142,39 @@ local function lex_string(context, str, start_pos, quote) end end +--[[- Lex the remainder of a zap escape sequence (`\z`). This consumes all leading +whitespace, and then continues lexing the string. + +@param context The current parser context. +@tparam string str The string we're lexing. +@tparam number pos The position to start lexing from. +@tparam number start_pos The actual start position of the string. +@tparam string quote The quote character, either " or '. +@treturn number The token id for strings. +@treturn number The new position. +@treturn nil A placeholder value. +@treturn table|nil The continuation function when the string is not finished. +]] +lex_string_zap = function(context, str, pos, start_pos, quote) + while true do + local next_pos, _, c = find(str, "([%S\r\n])", pos) + + if not next_pos then + context.report(errors.unfinished_string, start_pos, #str, quote) + return tokens.STRING, #str, nil, { lex_string_zap, 1, 1, quote } + end + + if c == "\n" or c == "\r" then + pos = newline(context, str, next_pos, c) + else + pos = next_pos + break + end + end + + return lex_string(context, str, pos, start_pos, quote) +end + --- Consume the start or end of a long string. -- @tparam string str The input string. -- @tparam number pos The start position. This must be after the first `[` or `]`. @@ -205,6 +228,45 @@ local function lex_long_str(context, str, start, len) end end +--[[- Lex the remainder of a long string. + +@param context The current parser context. +@tparam string str The string we're lexing. +@tparam number pos The position to start lexing from. +@tparam number start_pos The actual start position of the string. +@tparam number boundary_length The length of the boundary. +@treturn number The token id for strings. +@treturn number The new position. +@treturn nil A placeholder value. +@treturn table|nil The continuation function when the string is not finished. +]] +local function lex_long_string(context, str, pos, start_pos, boundary_length) + local end_pos = lex_long_str(context, str, pos, boundary_length) + if end_pos then return tokens.STRING, end_pos end + + context.report(errors.unfinished_long_string, start_pos, pos - 1, boundary_length) + return tokens.STRING, #str, nil, { lex_long_string, 0, 0, boundary_length } +end + +--[[- Lex the remainder of a long comment. + +@param context The current parser context. +@tparam string str The comment we're lexing. +@tparam number pos The position to start lexing from. +@tparam number start_pos The actual start position of the comment. +@tparam number boundary_length The length of the boundary. +@treturn number The token id for comments. +@treturn number The new position. +@treturn nil A placeholder value. +@treturn table|nil The continuation function when the comment is not finished. +]] +local function lex_long_comment(context, str, pos, start_pos, boundary_length) + local end_pos = lex_long_str(context, str, pos, boundary_length) + if end_pos then return tokens.COMMENT, end_pos end + + context.report(errors.unfinished_long_comment, start_pos, pos - 1, boundary_length) + return tokens.COMMENT, #str, nil, { lex_long_comment, 0, 0, boundary_length } +end --- Lex a single token, assuming we have removed all leading whitespace. -- @@ -229,16 +291,12 @@ local function lex_token(context, str, pos) elseif c >= "0" and c <= "9" then return lex_number(context, str, pos) -- Strings - elseif c == "\"" or c == "\'" then return lex_string(context, str, pos, c) + elseif c == "\"" or c == "\'" then return lex_string(context, str, pos + 1, pos, c) elseif c == "[" then local ok, boundary_pos = lex_long_str_boundary(str, pos + 1, "[") if ok then -- Long string - local end_pos = lex_long_str(context, str, boundary_pos + 1, boundary_pos - pos) - if end_pos then return tokens.STRING, end_pos end - - context.report(errors.unfinished_long_string, pos, boundary_pos, boundary_pos - pos) - return tokens.STRING, #str + return lex_long_string(context, str, boundary_pos + 1, pos, boundary_pos - pos) elseif pos + 1 == boundary_pos then -- Just a "[" return tokens.OSQUARE, pos else -- Malformed long string, for instance "[=" @@ -256,11 +314,7 @@ local function lex_token(context, str, pos) if sub(str, comment_pos, comment_pos) == "[" then local ok, boundary_pos = lex_long_str_boundary(str, comment_pos + 1, "[") if ok then - local end_pos = lex_long_str(context, str, boundary_pos + 1, boundary_pos - comment_pos) - if end_pos then return tokens.COMMENT, end_pos end - - context.report(errors.unfinished_long_comment, pos, boundary_pos, boundary_pos - comment_pos) - return tokens.COMMENT, #str + return lex_long_comment(context, str, boundary_pos + 1, pos, boundary_pos - comment_pos) end end @@ -357,8 +411,8 @@ local function lex_one(context, str, pos) elseif c == "\r" or c == "\n" then pos = newline(context, str, start_pos, c) else - local token_id, end_pos, content = lex_token(context, str, start_pos) - return token_id, start_pos, end_pos, content + local token_id, end_pos, content, continue = lex_token(context, str, start_pos) + return token_id, start_pos, end_pos, content, continue end end end diff --git a/projects/core/src/main/resources/data/computercraft/lua/rom/programs/edit.lua b/projects/core/src/main/resources/data/computercraft/lua/rom/programs/edit.lua index 748e58ad7..0b76f82f1 100644 --- a/projects/core/src/main/resources/data/computercraft/lua/rom/programs/edit.lua +++ b/projects/core/src/main/resources/data/computercraft/lua/rom/programs/edit.lua @@ -30,26 +30,23 @@ local x, y = 1, 1 local w, h = term.getSize() local scrollX, scrollY = 0, 0 -local tLines = {} +local tLines, tLineLexStates = {}, {} local bRunning = true -- Colours -local highlightColour, keywordColour, commentColour, textColour, bgColour, stringColour, errorColour -if term.isColour() then +local isColour = term.isColour() +local highlightColour, keywordColour, textColour, bgColour, errorColour +if isColour then bgColour = colours.black textColour = colours.white highlightColour = colours.yellow keywordColour = colours.yellow - commentColour = colours.green - stringColour = colours.red errorColour = colours.red else bgColour = colours.black textColour = colours.white highlightColour = colours.white keywordColour = colours.white - commentColour = colours.white - stringColour = colours.white errorColour = colours.white end @@ -100,6 +97,7 @@ local function load(_sPath) local sLine = file:read() while sLine do table.insert(tLines, sLine) + table.insert(tLineLexStates, false) sLine = file:read() end file:close() @@ -107,6 +105,7 @@ local function load(_sPath) if #tLines == 0 then table.insert(tLines, "") + table.insert(tLineLexStates, false) end end @@ -142,8 +141,9 @@ local tokens = require "cc.internal.syntax.parser".tokens local lex_one = require "cc.internal.syntax.lexer".lex_one local token_colours = { - [tokens.STRING] = stringColour, - [tokens.COMMENT] = commentColour, + [tokens.STRING] = isColour and colours.red or textColour, + [tokens.COMMENT] = isColour and colours.green or colours.lightGrey, + [tokens.NUMBER] = isColour and colours.magenta or textColour, -- Keywords [tokens.AND] = keywordColour, [tokens.BREAK] = keywordColour, @@ -175,26 +175,6 @@ end local lex_context = { line = function() end, report = function() end } -local function writeHighlighted(line) - local pos, colour = 1, nil - - while true do - local token, _, finish = lex_one(lex_context, line, pos) - if not token then break end - - local new_colour = token_colours[token] - if new_colour ~= colour then - term.setTextColor(new_colour) - colour = new_colour - end - - term.write(line:sub(pos, finish)) - pos = finish + 1 - end - - term.write(line:sub(pos)) -end - local tCompletions local nCompletion @@ -238,34 +218,94 @@ local function writeCompletion(sLine) end end -local function redrawText() - local cursorX, cursorY = x, y - for y = 1, h - 1 do - term.setCursorPos(1 - scrollX, y) +--- Check if two values are equal. If both values are lists, then the contents will be +-- checked for equality, to a depth of 1. +-- +-- @param x The first value. +-- @param x The second value. +-- @treturn boolean Whether the values are equal. +local function shallowEqual(x, y) + if x == y then return true end + + if type(x) ~= "table" or type(y) ~= "table" then return false end + if #x ~= #y then return false end + + for i = 1, #x do if x[i] ~= y[i] then return false end end + return true +end + +local function redrawLines(line, endLine) + if not endLine then endLine = line end + + local colour = term.getTextColour() + + -- Highlight all lines between line and endLine, highlighting further lines if their + -- lexer state has changed and aborting at the end of the screen. + local changed = false + while (changed or line <= endLine) and line - scrollY < h do + term.setCursorPos(1 - scrollX, line - scrollY) term.clearLine() - local sLine = tLines[y + scrollY] - if sLine ~= nil then - writeHighlighted(sLine) - if cursorY == y and cursorX == #sLine + 1 then - writeCompletion() + local contents = tLines[line] + if not contents then break end + + -- Lex our first token, either taking our continuation state (if present) or + -- the default lexer. + local pos, token, _, finish, continuation = 1 + local lex_state = tLineLexStates[line] + if lex_state then + token, finish, _, continuation = lex_state[1](lex_context, contents, table.unpack(lex_state, 2)) + else + token, _, finish, _, continuation = lex_one(lex_context, contents, 1) + end + + while token do + -- Print out that token + local new_colour = token_colours[token] + if new_colour ~= colour then + term.setTextColor(new_colour) + colour = new_colour end + term.write(contents:sub(pos, finish)) + + pos = finish + 1 + + -- If we have a continuation, then we've reached the end of the line. Abort. + if continuation then break end + + -- Otherwise lex another token and continue. + token, _, finish, _, continuation = lex_one(lex_context, contents, pos) + end + + -- Print the rest of the line. We don't strictly speaking need this, as it will + -- only ever contain whitespace. + term.write(contents:sub(pos)) + + if line == y and x == #contents + 1 then + writeCompletion() + colour = term.getTextColour() + end + + line = line + 1 + + -- Update the lext state of the next line. If that has changed, then + -- re-highlight it too. We store the continuation as nil rather than + -- false, to ensure we use the array part of the table. + if continuation == nil then continuation = false end + if tLineLexStates[line] ~= nil and not shallowEqual(tLineLexStates[line], continuation) then + tLineLexStates[line] = continuation or false + changed = true + else + changed = false end end + + term.setTextColor(colours.white) term.setCursorPos(x - scrollX, y - scrollY) end -local function redrawLine(_nY) - local sLine = tLines[_nY] - if sLine then - term.setCursorPos(1 - scrollX, _nY - scrollY) - term.clearLine() - writeHighlighted(sLine) - if _nY == y and x == #sLine + 1 then - writeCompletion() - end - term.setCursorPos(x - scrollX, _nY - scrollY) - end +local function redrawText() + redrawLines(scrollY + 1, scrollY + h - 1) end local function redrawMenu() @@ -462,12 +502,10 @@ local function setCursor(newX, newY) if bRedraw then redrawText() elseif y ~= oldY then - redrawLine(oldY) - redrawLine(y) + redrawLines(math.min(y, oldY), math.max(y, oldY)) else - redrawLine(y) + redrawLines(y) end - term.setCursorPos(screenX, screenY) redrawMenu() end @@ -522,7 +560,7 @@ while bRunning do if nCompletion < 1 then nCompletion = #tCompletions end - redrawLine(y) + redrawLines(y) elseif y > 1 then -- Move cursor up @@ -539,7 +577,7 @@ while bRunning do if nCompletion > #tCompletions then nCompletion = 1 end - redrawLine(y) + redrawLines(y) elseif y < #tLines then -- Move cursor down @@ -623,10 +661,11 @@ while bRunning do local sLine = tLines[y] tLines[y] = string.sub(sLine, 1, x - 1) .. string.sub(sLine, x + 1) recomplete() - redrawLine(y) + redrawLines(y) elseif y < #tLines then tLines[y] = tLines[y] .. tLines[y + 1] table.remove(tLines, y + 1) + table.remove(tLineLexStates, y + 1) recomplete() redrawText() end @@ -647,6 +686,7 @@ while bRunning do local sPrevLen = #tLines[y - 1] tLines[y - 1] = tLines[y - 1] .. tLines[y] table.remove(tLines, y) + table.remove(tLineLexStates, y) setCursor(sPrevLen + 1, y - 1) redrawText() end @@ -660,6 +700,7 @@ while bRunning do end tLines[y] = string.sub(sLine, 1, x - 1) table.insert(tLines, y + 1, string.rep(' ', spaces) .. string.sub(sLine, x)) + table.insert(tLineLexStates, y + 1, false) setCursor(spaces + 1, y + 1) redrawText()