xkb: treat pattern_white_space as whitespace

Author: mahkoh

compile-tests/testcases/t04/t060/t0463/expected.xkb

@@ -0,0 +1,21 @@
+xkb_keymap {
+    xkb_keycodes {
+        minimum = 8;
+        maximum = 255;
+
+        indicator 1 = "DUMMY";
+    };
+
+    xkb_types {
+        virtual_modifiers Dummy;
+    };
+
+    xkb_compat {
+        interpret VoidSymbol {
+            repeat = false;
+        };
+    };
+
+    xkb_symbols {
+    };
+};

compile-tests/testcases/t04/t060/t0463/input.xkb

@@ -0,0 +1,4 @@
+    	                
+               …        ‎                ‏                
                 

+//  ^tab    ^vertical tab   ^form feed   ^carriage return   ^next line  ^left-to-right mark   ^right-to-left mark   ^line separator   ^paragraph separator
+xkb_keymap { };

compile-tests/testcases/t04/t060/t0464/expected.toml

@@ -0,0 +1,5 @@
+keycodes = ""
+types = ""
+compat = ""
+symbols = "xyz"
+geometry = ""

compile-tests/testcases/t04/t060/t0464/input.toml

@@ -0,0 +1,4 @@
+rules = "a"
+
+[[groups]]
+layout = "X"

compile-tests/testcases/t04/t060/t0464/rules/a

@@ -0,0 +1,6 @@
+    	                
+               …        ‎                ‏                
                 

+//  ^tab    ^vertical tab   ^form feed   ^carriage return   ^next line  ^left-to-right mark   ^right-to-left mark   ^line separator   ^paragraph separator
+
+! layout[any] = symbols
+  *           = xyz

compose-tests/testcases/t00/t004/t0046/XCompose

@@ -0,0 +1,2 @@
+    	                                        …        ‎                ‏                
                 
                      <a>: X
+//  ^tab    ^vertical tab   ^form feed   ^carriage return   ^next line  ^left-to-right mark   ^right-to-left mark   ^line separator   ^paragraph separator

compose-tests/testcases/t00/t004/t0046/expected.txt

@@ -0,0 +1,3 @@
+a
+    composed
+        X

compose-tests/testcases/t00/t004/t0046/input.txt

@@ -0,0 +1 @@
+a

kbvm/release-notes.md

@@ -52,6 +52,21 @@
   ```
 - Fixed an arithmetic underflow when printing diagnostic messages containing
   non-ascii text.
+- The following characters are now always treated as insignificant whitespace
+  between tokens
+  - U+0009 - tab
+  - U+000b - vertical tab
+  - U+000c - form feed
+  - U+000d - carriage return
+  - U+0020 - space
+  - U+0085 - next line
+  - U+200e - left-to-right mark
+  - U+200f - right-to-left mark
+  - U+2028 - line separator
+  - U+2029 - paragraph separator
+  In xkb files, the following character is also insignificant whitespace
+  - U+000a - line feed
+  In RMLVO and compose files, line feeds are significant.
 
 # 0.1.3 (2025-02-13)
 

kbvm/src/xkb.rs

@@ -86,5 +86,6 @@ mod resolved;
 pub mod rmlvo;
 pub(crate) mod span;
 mod string_cooker;
+mod whitespace;
 #[cfg(feature = "x11")]
 pub mod x11;

kbvm/src/xkb/compose/lexer.rs

@@ -10,6 +10,7 @@ use {
         diagnostic::{DiagnosticKind, DiagnosticSink},
         interner::Interner,
         span::{SpanExt, SpanUnit, Spanned},
+        whitespace::consume_whitespace,
     },
     kbvm_proc::ad_hoc_display,
     std::sync::Arc,
@@ -111,13 +112,7 @@ impl LineLexer<'_, '_, '_> {
 
     fn lex_one(&mut self) -> Result<One, Spanned<LexerError>> {
         use LexerError::*;
-        while self.pos < self.code.len() {
-            if matches!(self.code[self.pos], b' ' | b'\t' | b'\r') {
-                self.pos += 1;
-            } else {
-                break;
-            }
-        }
+        consume_whitespace(&mut self.pos, &self.code, true);
         let mut b = match self.code.get(self.pos) {
             Some(c) => *c,
             _ => return Ok(One::Eof),

kbvm/src/xkb/kccgst/lexer.rs

@@ -14,6 +14,7 @@ use {
                 Token::{self, Float, Ident, Integer, KeyName, String},
             },
             span::{Span, SpanExt, SpanUnit, Spanned},
+            whitespace::consume_whitespace,
         },
     },
     std::{num::ParseFloatError, path::PathBuf, str::FromStr, sync::Arc},
@@ -125,13 +126,7 @@ impl ItemLexer<'_> {
         use LexerError::*;
         let mut b;
         loop {
-            while self.pos < self.code.len() {
-                if matches!(self.code[self.pos], b' ' | b'\t'..=b'\r') {
-                    self.pos += 1;
-                } else {
-                    break;
-                }
-            }
+            consume_whitespace(&mut self.pos, &self.code, false);
             match self.code.get(self.pos) {
                 Some(c) => b = *c,
                 _ => return Ok(None),

kbvm/src/xkb/rmlvo/lexer.rs

@@ -9,6 +9,7 @@ use {
         interner::Interner,
         rmlvo::token::Token,
         span::{SpanExt, SpanUnit, Spanned},
+        whitespace::consume_whitespace,
     },
     std::{path::PathBuf, sync::Arc},
     thiserror::Error,
@@ -104,13 +105,7 @@ impl LineLexer<'_> {
         use LexerError::*;
         let mut b;
         loop {
-            while self.pos < self.code.len() {
-                if matches!(self.code[self.pos], b' ' | b'\t' | b'\r') {
-                    self.pos += 1;
-                } else {
-                    break;
-                }
-            }
+            consume_whitespace(&mut self.pos, &self.code, true);
             b = match self.code.get(self.pos) {
                 Some(c) => *c,
                 _ => return Ok(One::Eof),

kbvm/src/xkb/whitespace.rs

@@ -0,0 +1,51 @@
+pub(crate) fn consume_whitespace(pos: &mut usize, c: &[u8], significant_newline: bool) {
+    let mut p = *pos;
+    while p < c.len() {
+        let b0 = c[p];
+        match b0 {
+            0x0020 | 0x0009 => {
+                // U+0020 - space
+                // U+0009 - tab
+                p += 1;
+                continue;
+            }
+            0x000a if significant_newline => {
+                // U+000a - line feed
+                break;
+            }
+            0x000a..=0x000d => {
+                // U+000a - line feed
+                // U+000b - vertical tab
+                // U+000c - form feed
+                // U+000d - carriage return
+                p += 1;
+                continue;
+            }
+            0xc2 if p + 1 < c.len() => {
+                let b1 = c[p + 1];
+                if b1 == 0x85 {
+                    // U+0085 - next line
+                    p += 2;
+                    continue;
+                }
+            }
+            0xe2 if p + 2 < c.len() => {
+                let b1 = c[p + 1];
+                if b1 == 0x80 {
+                    let b2 = c[p + 2];
+                    if matches!(b2, 0x8e | 0x8f | 0xa8 | 0xa9) {
+                        // U+200e - left-to-right mark
+                        // U+200f - right-to-left mark
+                        // U+2028 - line separator
+                        // U+2029 - paragraph separator
+                        p += 3;
+                        continue;
+                    }
+                }
+            }
+            _ => {}
+        }
+        break;
+    }
+    *pos = p;
+}