Syntax highlighting for multiline tokens in edit

I don't love the implementation of this (see discussion in #2220), but
it's better than nothing. Wow, the editor really needs a bit of a
rewrite, the code is kinda messy.

Fixes #1396.

projects/core/src/main/resources/data/computercraft/lua/rom/modules/main/cc/internal/syntax/lexer.lua

@@ -101,16 +101,21 @@ local function lex_number(context, str, start)
     return tokens.NUMBER, pos - 1
 end
 
---- Lex a quoted string.
+local lex_string_zap
---
+
--- @param context The current parser context.
+--[[- Lex a quoted string.
--- @tparam string str The string we're lexing.
+
--- @tparam number start_pos The start position of the string.
+@param context The current parser context.
--- @tparam string quote The quote character, either " or '.
+@tparam string str The string we're lexing.
--- @treturn number The token id for strings.
+@tparam number pos The position to start lexing from.
--- @treturn number The new position.
+@tparam number start_pos The actual start position of the string.
-local function lex_string(context, str, start_pos, quote)
+@tparam string quote The quote character, either " or '.
-    local pos = start_pos + 1
+@treturn number The token id for strings.
+@treturn number The new position.
+@treturn nil A placeholder value.
+@treturn table|nil The continuation function when the string is not finished.
+]]
+local function lex_string(context, str, pos, start_pos, quote)
     while true do
         local c = sub(str, pos, pos)
         if c == quote then
@@ -125,24 +130,9 @@ local function lex_string(context, str, start_pos, quote)
                 pos = newline(context, str, pos + 1, c)
             elseif c == "" then
                 context.report(errors.unfinished_string_escape, start_pos, pos, quote)
-                return tokens.STRING, pos
+                return tokens.STRING, pos, nil, { lex_string, 1, 1, quote }
             elseif c == "z" then
-                pos = pos + 2
+                return lex_string_zap(context, str, pos + 2, start_pos, quote)
-                while true do
-                    local next_pos, _, c  = find(str, "([%S\r\n])", pos)
-
-                    if not next_pos then
-                        context.report(errors.unfinished_string, start_pos, #str, quote)
-                        return tokens.STRING, #str
-                    end
-
-                    if c == "\n" or c == "\r" then
-                        pos = newline(context, str, next_pos, c)
-                    else
-                        pos = next_pos
-                        break
-                    end
-                end
             else
                 pos = pos + 2
             end
@@ -152,6 +142,39 @@ local function lex_string(context, str, start_pos, quote)
     end
 end
 
+--[[- Lex the remainder of a zap escape sequence (`\z`). This consumes all leading
+whitespace, and then continues lexing the string.
+
+@param context The current parser context.
+@tparam string str The string we're lexing.
+@tparam number pos The position to start lexing from.
+@tparam number start_pos The actual start position of the string.
+@tparam string quote The quote character, either " or '.
+@treturn number The token id for strings.
+@treturn number The new position.
+@treturn nil A placeholder value.
+@treturn table|nil The continuation function when the string is not finished.
+]]
+lex_string_zap = function(context, str, pos, start_pos, quote)
+    while true do
+        local next_pos, _, c  = find(str, "([%S\r\n])", pos)
+
+        if not next_pos then
+            context.report(errors.unfinished_string, start_pos, #str, quote)
+            return tokens.STRING, #str, nil, { lex_string_zap, 1, 1, quote }
+        end
+
+        if c == "\n" or c == "\r" then
+            pos = newline(context, str, next_pos, c)
+        else
+            pos = next_pos
+            break
+        end
+    end
+
+    return lex_string(context, str, pos, start_pos, quote)
+end
+
 --- Consume the start or end of a long string.
 -- @tparam string str The input string.
 -- @tparam number pos The start position. This must be after the first `[` or `]`.
@@ -205,6 +228,45 @@ local function lex_long_str(context, str, start, len)
     end
 end
 
+--[[- Lex the remainder of a long string.
+
+@param context The current parser context.
+@tparam string str The string we're lexing.
+@tparam number pos The position to start lexing from.
+@tparam number start_pos The actual start position of the string.
+@tparam number boundary_length The length of the boundary.
+@treturn number The token id for strings.
+@treturn number The new position.
+@treturn nil A placeholder value.
+@treturn table|nil The continuation function when the string is not finished.
+]]
+local function lex_long_string(context, str, pos, start_pos, boundary_length)
+    local end_pos = lex_long_str(context, str, pos, boundary_length)
+    if end_pos then return tokens.STRING, end_pos end
+
+    context.report(errors.unfinished_long_string, start_pos, pos - 1, boundary_length)
+    return tokens.STRING, #str, nil, { lex_long_string, 0, 0, boundary_length }
+end
+
+--[[- Lex the remainder of a long comment.
+
+@param context The current parser context.
+@tparam string str The comment we're lexing.
+@tparam number pos The position to start lexing from.
+@tparam number start_pos The actual start position of the comment.
+@tparam number boundary_length The length of the boundary.
+@treturn number The token id for comments.
+@treturn number The new position.
+@treturn nil A placeholder value.
+@treturn table|nil The continuation function when the comment is not finished.
+]]
+local function lex_long_comment(context, str, pos, start_pos, boundary_length)
+    local end_pos = lex_long_str(context, str, pos, boundary_length)
+    if end_pos then return tokens.COMMENT, end_pos end
+
+    context.report(errors.unfinished_long_comment, start_pos, pos - 1, boundary_length)
+    return tokens.COMMENT, #str, nil, { lex_long_comment, 0, 0, boundary_length }
+end
 
 --- Lex a single token, assuming we have removed all leading whitespace.
 --
@@ -229,16 +291,12 @@ local function lex_token(context, str, pos)
     elseif c >= "0" and c <= "9" then return lex_number(context, str, pos)
 
     -- Strings
-    elseif c == "\"" or c == "\'" then return lex_string(context, str, pos, c)
+    elseif c == "\"" or c == "\'" then return lex_string(context, str, pos + 1, pos, c)
 
     elseif c == "[" then
         local ok, boundary_pos = lex_long_str_boundary(str, pos + 1, "[")
         if ok then -- Long string
-            local end_pos = lex_long_str(context, str, boundary_pos + 1, boundary_pos - pos)
+            return lex_long_string(context, str, boundary_pos + 1, pos, boundary_pos - pos)
-            if end_pos then return tokens.STRING, end_pos end
-
-            context.report(errors.unfinished_long_string, pos, boundary_pos, boundary_pos - pos)
-            return tokens.STRING, #str
         elseif pos + 1 == boundary_pos then -- Just a "["
             return tokens.OSQUARE, pos
         else -- Malformed long string, for instance "[="
@@ -256,11 +314,7 @@ local function lex_token(context, str, pos)
         if sub(str, comment_pos, comment_pos) == "[" then
             local ok, boundary_pos = lex_long_str_boundary(str, comment_pos + 1, "[")
             if ok then
-                local end_pos = lex_long_str(context, str, boundary_pos + 1, boundary_pos - comment_pos)
+                return lex_long_comment(context, str, boundary_pos + 1, pos, boundary_pos - comment_pos)
-                if end_pos then return tokens.COMMENT, end_pos end
-
-                context.report(errors.unfinished_long_comment, pos, boundary_pos, boundary_pos - comment_pos)
-                return tokens.COMMENT, #str
             end
         end
 
@@ -357,8 +411,8 @@ local function lex_one(context, str, pos)
         elseif c == "\r" or c == "\n" then
             pos = newline(context, str, start_pos, c)
         else
-            local token_id, end_pos, content = lex_token(context, str, start_pos)
+            local token_id, end_pos, content, continue = lex_token(context, str, start_pos)
-            return token_id, start_pos, end_pos, content
+            return token_id, start_pos, end_pos, content, continue
         end
     end
 end

projects/core/src/main/resources/data/computercraft/lua/rom/programs/edit.lua

@@ -30,26 +30,23 @@ local x, y = 1, 1
 local w, h = term.getSize()
 local scrollX, scrollY = 0, 0
 
-local tLines = {}
+local tLines, tLineLexStates = {}, {}
 local bRunning = true
 
 -- Colours
-local highlightColour, keywordColour, commentColour, textColour, bgColour, stringColour, errorColour
+local isColour = term.isColour()
-if term.isColour() then
+local highlightColour, keywordColour, textColour, bgColour, errorColour
+if isColour then
     bgColour = colours.black
     textColour = colours.white
     highlightColour = colours.yellow
     keywordColour = colours.yellow
-    commentColour = colours.green
-    stringColour = colours.red
     errorColour = colours.red
 else
     bgColour = colours.black
     textColour = colours.white
     highlightColour = colours.white
     keywordColour = colours.white
-    commentColour = colours.white
-    stringColour = colours.white
     errorColour = colours.white
 end
 
@@ -100,6 +97,7 @@ local function load(_sPath)
         local sLine = file:read()
         while sLine do
             table.insert(tLines, sLine)
+            table.insert(tLineLexStates, false)
             sLine = file:read()
         end
         file:close()
@@ -107,6 +105,7 @@ local function load(_sPath)
 
     if #tLines == 0 then
         table.insert(tLines, "")
+        table.insert(tLineLexStates, false)
     end
 end
 
@@ -142,8 +141,9 @@ local tokens = require "cc.internal.syntax.parser".tokens
 local lex_one = require "cc.internal.syntax.lexer".lex_one
 
 local token_colours = {
-    [tokens.STRING] = stringColour,
+    [tokens.STRING] = isColour and colours.red or textColour,
-    [tokens.COMMENT] = commentColour,
+    [tokens.COMMENT] = isColour and colours.green or colours.lightGrey,
+    [tokens.NUMBER] = isColour and colours.magenta or textColour,
     -- Keywords
     [tokens.AND] = keywordColour,
     [tokens.BREAK] = keywordColour,
@@ -175,26 +175,6 @@ end
 
 local lex_context = { line = function() end, report = function() end }
 
-local function writeHighlighted(line)
-    local pos, colour = 1, nil
-
-    while true do
-        local token, _, finish = lex_one(lex_context, line, pos)
-        if not token then break end
-
-        local new_colour = token_colours[token]
-        if new_colour ~= colour then
-            term.setTextColor(new_colour)
-            colour = new_colour
-        end
-
-        term.write(line:sub(pos, finish))
-        pos = finish + 1
-    end
-
-    term.write(line:sub(pos))
-end
-
 local tCompletions
 local nCompletion
 
@@ -238,34 +218,94 @@ local function writeCompletion(sLine)
     end
 end
 
-local function redrawText()
+--- Check if two values are equal. If both values are lists, then the contents will be
-    local cursorX, cursorY = x, y
+-- checked for equality, to a depth of 1.
-    for y = 1, h - 1 do
+--
-        term.setCursorPos(1 - scrollX, y)
+-- @param x The first value.
+-- @param x The second value.
+-- @treturn boolean Whether the values are equal.
+local function shallowEqual(x, y)
+    if x == y then return true end
+
+    if type(x) ~= "table" or type(y) ~= "table" then return false end
+    if #x ~= #y then return false end
+
+    for i = 1, #x do if x[i] ~= y[i] then return false end end
+    return true
+end
+
+local function redrawLines(line, endLine)
+    if not endLine then endLine = line end
+
+    local colour = term.getTextColour()
+
+    -- Highlight all lines between line and endLine, highlighting further lines if their
+    -- lexer state has changed and aborting at the end of the screen.
+    local changed = false
+    while (changed or line <= endLine) and line - scrollY < h do
+        term.setCursorPos(1 - scrollX, line - scrollY)
         term.clearLine()
 
-        local sLine = tLines[y + scrollY]
+        local contents = tLines[line]
-        if sLine ~= nil then
+        if not contents then break end
-            writeHighlighted(sLine)
+
-            if cursorY == y and cursorX == #sLine + 1 then
+        -- Lex our first token, either taking our continuation state (if present) or
-                writeCompletion()
+        -- the default lexer.
+        local pos, token, _, finish, continuation = 1
+        local lex_state = tLineLexStates[line]
+        if lex_state then
+            token, finish, _, continuation = lex_state[1](lex_context, contents, table.unpack(lex_state, 2))
+        else
+            token, _, finish, _, continuation = lex_one(lex_context, contents, 1)
+        end
+
+        while token do
+            -- Print out that token
+            local new_colour = token_colours[token]
+            if new_colour ~= colour then
+                term.setTextColor(new_colour)
+                colour = new_colour
             end
+            term.write(contents:sub(pos, finish))
+
+            pos = finish + 1
+
+            -- If we have a continuation, then we've reached the end of the line. Abort.
+            if continuation then break end
+
+            -- Otherwise lex another token and continue.
+            token, _, finish, _, continuation = lex_one(lex_context, contents, pos)
+        end
+
+        -- Print the rest of the line. We don't strictly speaking need this, as it will
+        -- only ever contain whitespace.
+        term.write(contents:sub(pos))
+
+        if line == y and x == #contents + 1 then
+            writeCompletion()
+            colour = term.getTextColour()
+        end
+
+        line = line + 1
+
+        -- Update the lext state of the next line. If that has changed, then
+        -- re-highlight it too. We store the continuation as nil rather than
+        -- false, to ensure we use the array part of the table.
+        if continuation == nil then continuation = false end
+        if tLineLexStates[line] ~= nil and not shallowEqual(tLineLexStates[line], continuation) then
+            tLineLexStates[line] = continuation or false
+            changed = true
+        else
+            changed = false
         end
     end
+
+    term.setTextColor(colours.white)
     term.setCursorPos(x - scrollX, y - scrollY)
 end
 
-local function redrawLine(_nY)
+local function redrawText()
-    local sLine = tLines[_nY]
+    redrawLines(scrollY + 1, scrollY + h - 1)
-    if sLine then
-        term.setCursorPos(1 - scrollX, _nY - scrollY)
-        term.clearLine()
-        writeHighlighted(sLine)
-        if _nY == y and x == #sLine + 1 then
-            writeCompletion()
-        end
-        term.setCursorPos(x - scrollX, _nY - scrollY)
-    end
 end
 
 local function redrawMenu()
@@ -462,12 +502,10 @@ local function setCursor(newX, newY)
     if bRedraw then
         redrawText()
     elseif y ~= oldY then
-        redrawLine(oldY)
+        redrawLines(math.min(y, oldY), math.max(y, oldY))
-        redrawLine(y)
     else
-        redrawLine(y)
+        redrawLines(y)
     end
-    term.setCursorPos(screenX, screenY)
 
     redrawMenu()
 end
@@ -522,7 +560,7 @@ while bRunning do
                     if nCompletion < 1 then
                         nCompletion = #tCompletions
                     end
-                    redrawLine(y)
+                    redrawLines(y)
 
                 elseif y > 1 then
                     -- Move cursor up
@@ -539,7 +577,7 @@ while bRunning do
                     if nCompletion > #tCompletions then
                         nCompletion = 1
                     end
-                    redrawLine(y)
+                    redrawLines(y)
 
                 elseif y < #tLines then
                     -- Move cursor down
@@ -623,10 +661,11 @@ while bRunning do
                     local sLine = tLines[y]
                     tLines[y] = string.sub(sLine, 1, x - 1) .. string.sub(sLine, x + 1)
                     recomplete()
-                    redrawLine(y)
+                    redrawLines(y)
                 elseif y < #tLines then
                     tLines[y] = tLines[y] .. tLines[y + 1]
                     table.remove(tLines, y + 1)
+                    table.remove(tLineLexStates, y + 1)
                     recomplete()
                     redrawText()
                 end
@@ -647,6 +686,7 @@ while bRunning do
                     local sPrevLen = #tLines[y - 1]
                     tLines[y - 1] = tLines[y - 1] .. tLines[y]
                     table.remove(tLines, y)
+                    table.remove(tLineLexStates, y)
                     setCursor(sPrevLen + 1, y - 1)
                     redrawText()
                 end
@@ -660,6 +700,7 @@ while bRunning do
                 end
                 tLines[y] = string.sub(sLine, 1, x - 1)
                 table.insert(tLines, y + 1, string.rep(' ', spaces) .. string.sub(sLine, x))
+                table.insert(tLineLexStates, y + 1, false)
                 setCursor(spaces + 1, y + 1)
                 redrawText()