From b37710e474a805a9a5a7f5b25a084277c73b7fb3 Mon Sep 17 00:00:00 2001 From: immerrr Date: Mon, 21 Sep 2020 16:50:46 +0200 Subject: [PATCH 1/2] Enable fontification of variable names in "for" and function parameters - drop make-delimited-matcher magical function - add regression test for issue #163 --- lua-mode.el | 180 +++++++++++++++-------------------------- test/test-font-lock.el | 67 ++++++++++++--- 2 files changed, 119 insertions(+), 128 deletions(-) diff --git a/lua-mode.el b/lua-mode.el index f1640b9..4b682d6 100644 --- a/lua-mode.el +++ b/lua-mode.el @@ -138,7 +138,17 @@ (lua-keyword (symbol "break" "do" "else" "elseif" "end" "for" "function" "goto" "if" "in" "local" "repeat" "return" - "then" "until" "while")))) + "then" "until" "while")) + (lua-up-to-9-variables + (seq (group-n 1 lua-name) ws + (? "," ws (group-n 2 lua-name) ws + (? "," ws (group-n 3 lua-name) ws + (? "," ws (group-n 4 lua-name) ws + (? "," ws (group-n 5 lua-name) ws + (? "," ws (group-n 6 lua-name) ws + (? "," ws (group-n 7 lua-name) ws + (? "," ws (group-n 8 lua-name) ws + (? "," ws (group-n 9 lua-name) ws)))))))))))) (defmacro lua-rx (&rest regexps) (eval `(rx-let ,lua--rx-bindings @@ -223,7 +233,17 @@ element is itself expanded with `lua-rx-to-string'. " (lua-keyword :rx (symbol "break" "do" "else" "elseif" "end" "for" "function" "goto" "if" "in" "local" "repeat" "return" - "then" "until" "while"))))))) + "then" "until" "while")) + (lua-up-to-9-variables + :rx (seq (group-n 1 lua-name) ws + (? "," ws (group-n 2 lua-name) ws + (? "," ws (group-n 3 lua-name) ws + (? "," ws (group-n 4 lua-name) ws + (? "," ws (group-n 5 lua-name) ws + (? "," ws (group-n 6 lua-name) ws + (? "," ws (group-n 7 lua-name) ws + (? "," ws (group-n 8 lua-name) ws + (? "," ws (group-n 9 lua-name) ws))))))))))))))) ;; Local variables @@ -534,86 +554,6 @@ traceback location." This is a compilation of 5.1, 5.2 and 5.3 builtins taken from the index of respective Lua reference manuals.") -(eval-and-compile - (defun lua-make-delimited-matcher (elt-regexp sep-regexp end-regexp) - "Construct matcher function for `font-lock-keywords' to match a sequence. - -It's supposed to match sequences with following EBNF: - -ELT-REGEXP { SEP-REGEXP ELT-REGEXP } END-REGEXP - -The sequence is parsed one token at a time. If non-nil is -returned, `match-data' will have one or more of the following -groups set according to next matched token: - -1. matched element token -2. unmatched garbage characters -3. misplaced token (i.e. SEP-REGEXP when ELT-REGEXP is expected) -4. matched separator token -5. matched end token - -Blanks & comments between tokens are silently skipped. -Groups 6-9 can be used in any of argument regexps." - (let* - ((delimited-matcher-re-template - "\\=\\(?2:.*?\\)\\(?:\\(?%s:\\(?4:%s\\)\\|\\(?5:%s\\)\\)\\|\\(?%s:\\(?1:%s\\)\\)\\)") - ;; There's some magic to this regexp. It works as follows: - ;; - ;; A. start at (point) - ;; B. non-greedy match of garbage-characters (?2:) - ;; C. try matching separator (?4:) or end-token (?5:) - ;; D. try matching element (?1:) - ;; - ;; Simple, but there's a trick: pt.C and pt.D are embraced by one more - ;; group whose purpose is determined only after the template is - ;; formatted (?%s:): - ;; - ;; - if element is expected, then D's parent group becomes "shy" and C's - ;; parent becomes group 3 (aka misplaced token), so if D matches when - ;; an element is expected, it'll be marked with warning face. - ;; - ;; - if separator-or-end-token is expected, then it's the opposite: - ;; C's parent becomes shy and D's will be matched as misplaced token. - (elt-expected-re (format delimited-matcher-re-template - 3 sep-regexp end-regexp "" elt-regexp)) - (sep-or-end-expected-re (format delimited-matcher-re-template - "" sep-regexp end-regexp 3 elt-regexp))) - - (lambda (end) - (let* ((prev-elt-p (match-beginning 1)) - (prev-end-p (match-beginning 5)) - - (regexp (if prev-elt-p sep-or-end-expected-re elt-expected-re)) - (comment-start (lua-comment-start-pos (syntax-ppss))) - (parse-stop end)) - - ;; If token starts inside comment, or end-token was encountered, stop. - (when (and (not comment-start) - (not prev-end-p)) - ;; Skip all comments & whitespace. forward-comment doesn't have boundary - ;; argument, so make sure point isn't beyond parse-stop afterwards. - (while (and (< (point) end) - (forward-comment 1))) - (goto-char (min (point) parse-stop)) - - ;; Reuse comment-start variable to store beginning of comment that is - ;; placed before line-end-position so as to make sure token search doesn't - ;; enter that comment. - (setq comment-start - (lua-comment-start-pos - (save-excursion - (parse-partial-sexp (point) parse-stop - nil nil nil 'stop-inside-comment))) - parse-stop (or comment-start parse-stop)) - - ;; Now, let's match stuff. If regular matcher fails, declare a span of - ;; non-blanks 'garbage', and the next iteration will start from where the - ;; garbage ends. If couldn't match any garbage, move point to the end - ;; and return nil. - (or (re-search-forward regexp parse-stop t) - (re-search-forward "\\(?1:\\(?2:[^ \t]+\\)\\)" parse-stop 'skip) - (prog1 nil (goto-char end))))))))) - (defvar lua-font-lock-keywords `(;; highlight the hash-bang line "#!/foo/bar/lua" as comment @@ -641,42 +581,50 @@ Groups 6-9 can be used in any of argument regexps." (,lua--builtins (1 font-lock-builtin-face) (2 font-lock-builtin-face nil noerror)) - ("^[ \t]*\\_" - (,(lua-make-delimited-matcher (lua-rx lua-name) "," - (lua-rx (or (symbol "in") lua-assignment-op))) - nil nil - (1 font-lock-variable-name-face nil noerror) - (2 font-lock-warning-face t noerror) - (3 font-lock-warning-face t noerror))) - - ;; Handle local variable/function names - ;; local blalba, xyzzy = - ;; ^^^^^^ ^^^^^ - ;; - ;; local function foobar(x,y,z) - ;; ^^^^^^ - ;; local foobar = function(x,y,z) - ;; ^^^^^^ - ("^[ \t]*\\_" - (0 font-lock-keyword-face) - - ;; (* nonl) at the end is to consume trailing characters or otherwise they - ;; delimited matcher would attempt to parse them afterwards and wrongly - ;; highlight parentheses as incorrect variable name characters. - (,(lua-rx point ws lua-funcheader (* nonl)) - nil nil - (1 font-lock-function-name-face nil noerror)) - - (,(lua-make-delimited-matcher (lua-rx lua-name) "," - (lua-rx lua-assignment-op)) - nil nil - (1 font-lock-variable-name-face nil noerror) - (2 font-lock-warning-face t noerror) - (3 font-lock-warning-face t noerror))) - - (,(lua-rx (or bol ";") ws lua-funcheader) + (,(lua-rx (symbol "for") ws+ lua-up-to-9-variables) + (1 font-lock-variable-name-face) + (2 font-lock-variable-name-face nil noerror) + (3 font-lock-variable-name-face nil noerror) + (4 font-lock-variable-name-face nil noerror) + (5 font-lock-variable-name-face nil noerror) + (6 font-lock-variable-name-face nil noerror) + (7 font-lock-variable-name-face nil noerror) + (8 font-lock-variable-name-face nil noerror) + (9 font-lock-variable-name-face nil noerror)) + + (,(lua-rx (symbol "function") (? ws+ lua-funcname) ws "(" ws lua-up-to-9-variables) + (1 font-lock-variable-name-face) + (2 font-lock-variable-name-face nil noerror) + (3 font-lock-variable-name-face nil noerror) + (4 font-lock-variable-name-face nil noerror) + (5 font-lock-variable-name-face nil noerror) + (6 font-lock-variable-name-face nil noerror) + (7 font-lock-variable-name-face nil noerror) + (8 font-lock-variable-name-face nil noerror) + (9 font-lock-variable-name-face nil noerror)) + + (,(lua-rx lua-funcheader) (1 font-lock-function-name-face)) + ;; local x, y, z + ;; local x = ..... + ;; + ;; NOTE: this is intentionally below funcheader matcher, so that in + ;; + ;; local foo = function() ... + ;; + ;; "foo" is fontified as function-name-face, and variable-name-face is not applied. + (,(lua-rx (symbol "local") ws+ lua-up-to-9-variables) + (1 font-lock-variable-name-face) + (2 font-lock-variable-name-face nil noerror) + (3 font-lock-variable-name-face nil noerror) + (4 font-lock-variable-name-face nil noerror) + (5 font-lock-variable-name-face nil noerror) + (6 font-lock-variable-name-face nil noerror) + (7 font-lock-variable-name-face nil noerror) + (8 font-lock-variable-name-face nil noerror) + (9 font-lock-variable-name-face nil noerror)) + (,(lua-rx (or (group-n 1 "@" (symbol "author" "copyright" "field" "release" "return" "see" "usage" "description")) diff --git a/test/test-font-lock.el b/test/test-font-lock.el index f30b1fc..5b9cfce 100644 --- a/test/test-font-lock.el +++ b/test/test-font-lock.el @@ -68,10 +68,9 @@ _table.sort(foobar) (expect "_do foo(5) end_" :to-be-fontified-as - '(nil)) - ) + '(nil))) (it "fontifies keywords used as attributes" - ;; Hint user that keywords cannot be used like that + ;; Hint user that keywords cannot be used like that (expect "foo(5).end" :to-be-fontified-as '(("end" keyword))) @@ -79,20 +78,64 @@ _table.sort(foobar) :to-be-fontified-as '(("end" keyword))))) +(describe "Fontification of variables" + (it "fontifies \"local foo, bar, baz = 1, 2, 3\"" + (expect "local foo,bar , baz = 1, 2, 3" + :to-be-fontified-as '(("local" keyword "foo" variable-name "bar" variable-name "baz" variable-name)))) + + (it "fontifies \"local foo, bar, baz\"" + (expect "local foo,bar , baz" + :to-be-fontified-as '(("local" keyword "foo" variable-name "bar" variable-name "baz" variable-name)))) + + (it "fontifies \"local x =\" at end of buffer" + (expect "local x =" + :to-be-fontified-as '(("local" keyword "x" variable-name)))) + + (it "fontifies local \"x =\" at end of line" + ;; Issue #163 + (expect "local x = + +foo = bar" + :to-be-fontified-as '(("local" keyword "x" variable-name) + () + ()))) + + (it "fontifies \"for x123 =\"" + (expect "for x123 =" + :to-be-fontified-as '(("for" keyword "x123" variable-name)))) + + (it "fontifies \"for x, y, z\"" + (expect "for x, y, z in " + :to-be-fontified-as '(("for" keyword "x" variable-name "y" variable-name "z" variable-name + "in" keyword))))) + (describe "Fontification of function headers" (it "fontifies function (...) headers" - (expect "function bar() end" - :to-be-fontified-as '(("function" keyword "bar" function-name "end" keyword)))) + (expect "function bar(x, y) end" + :to-be-fontified-as '(("function" keyword "bar" function-name + "x" variable-name "y" variable-name + "end" keyword)))) (it "fontifies local function (...) headers" - (expect "local function baz() end" - :to-be-fontified-as '(("local" keyword "function" keyword "baz" function-name "end" keyword)))) + (expect "local function baz(x, y) end" + :to-be-fontified-as '(("local" keyword "function" keyword "baz" function-name + "x" variable-name "y" variable-name + "end" keyword)))) (it "fontifies = function (...) headers" - (expect "qux = function() end" - :to-be-fontified-as '(("qux" function-name "function" keyword "end" keyword)))) + (expect "qux = function(x, y) end" + :to-be-fontified-as '(("qux" function-name "function" keyword + "x" variable-name "y" variable-name + "end" keyword)))) (it "fontifies local = function (...) headers" - (expect "local quux = function() end" - :to-be-fontified-as '(("local" keyword "quux" function-name "function" keyword "end" keyword)))) + (expect "local quux = function(x, y) end" + :to-be-fontified-as '(("local" keyword "quux" function-name "function" keyword + "x" variable-name "y" variable-name + "end" keyword)))) + + (it "fontifies parameters in function literals" + (expect "foo(function(x, y))" + :to-be-fontified-as '(("function" keyword + "x" variable-name "y" variable-name)))) (it "fontifies different variations of headers altogether" (expect @@ -147,7 +190,7 @@ end" ("end" keyword)))) (it "does not choke on function names with underscores" - (expect + (expect ;; Check all defun variants, check embedded defuns "\ function foo() From 9c275cd3cd1b4a14e3079adf223d616bb452e057 Mon Sep 17 00:00:00 2001 From: immerrr Date: Mon, 21 Sep 2020 16:52:45 +0200 Subject: [PATCH 2/2] Add regression test for issue #157, fix lua-get-line-faces --- test/test-font-lock.el | 9 +++++++++ test/utils.el | 24 +++++++++++++----------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/test/test-font-lock.el b/test/test-font-lock.el index 5b9cfce..8fbb974 100644 --- a/test/test-font-lock.el +++ b/test/test-font-lock.el @@ -100,6 +100,15 @@ foo = bar" () ()))) + (it "does not fontify \"for\" inside strings" + ;; Issue #157 + (expect "local xx = [[ +for abc def +]]" + :to-be-fontified-as '(("local" keyword "xx" variable-name "[[" string) + ("for abc def" string) + ("]]" string)))) + (it "fontifies \"for x123 =\"" (expect "for x123 =" :to-be-fontified-as '(("for" keyword "x123" variable-name)))) diff --git a/test/utils.el b/test/utils.el index 4192db0..6b05737 100644 --- a/test/utils.el +++ b/test/utils.el @@ -56,16 +56,17 @@ E.g. for properly fontified Lua string \"local x = 100\" it should return \"x\" font-lock-variable-name-face \"100\" font-lock-constant-face) " - (let ((pos 0) - nextpos - result prop newprop) - (while pos - (setq nextpos (next-property-change pos str) - newprop (or (get-text-property pos 'face str) - (get-text-property pos 'font-lock-face str))) + (let* ((pos 0) + (prop (or (get-text-property pos 'face str) + (get-text-property pos 'font-lock-face str))) + (nextpos 0) + newprop + result) + (while nextpos + (setq nextpos (next-property-change nextpos str)) + (setq newprop (when nextpos (or (get-text-property nextpos 'face str) + (get-text-property nextpos 'font-lock-face str)))) (when (not (equal prop newprop)) - (setq prop newprop) - (when (listp prop) (when (eq (car-safe (last prop)) 'default) (setq prop (butlast prop))) @@ -76,8 +77,9 @@ E.g. for properly fontified Lua string \"local x = 100\" it should return (setq prop nil)))) (when prop (push (substring-no-properties str pos nextpos) result) - (push prop result))) - (setq pos nextpos)) + (push prop result)) + (setq prop newprop + pos nextpos))) (nreverse result))) (defun lua-fontify-str (str)