WIP pre-parser

packages/parser/src/modules.rs

@@ -83,7 +83,7 @@ impl ImportPathDecl {
 						(result, _) = namespace.as_inner();
 					} else {
 						let (next, _) = namespace.as_inner();
-						result = utils::join_substrs_unchecked(&result, &next);
+						result = utils::join_substrs(&result, &next);
 					}
 
 					import_path = path.as_ref();
@@ -93,7 +93,7 @@ impl ImportPathDecl {
 						(result, _) = name.as_inner();
 					} else {
 						let (next, _) = name.as_inner();
-						result = utils::join_substrs_unchecked(&result, &next);
+						result = utils::join_substrs(&result, &next);
 					}
 
 					break;

packages/parser/src/utils/mod.rs

@@ -103,7 +103,7 @@ impl ToRange for Span {
 ///
 /// - `start` and `end` point to different `ArcStr` allocations
 /// - `end.range().end` is less than or equal to `start.range().start`
-pub(crate) fn join_substrs_unchecked(start: &Substr, end: &Substr) -> Substr {
+pub fn join_substrs(start: &Substr, end: &Substr) -> Substr {
 	assert!(ArcStr::ptr_eq(start.parent(), end.parent()));
 	assert!(end.range().end > start.range().start);
 

packages/server/src/documents.rs

@@ -25,6 +25,7 @@ use ropey::Rope;
 use crate::{
 	ipc::{notify, Ipc},
 	lsp_extensions::{UnreadDependency, UnreadDependencyParams},
+	pre,
 	references::TokenReferences,
 	utils::{self, Canonicalize},
 	workspace::Workspace,
@@ -102,6 +103,9 @@ pub struct WgslDocumentBundle {
 
 impl WgslDocumentBundle {
 	pub fn new<S: Into<ArcStr>>(source: S, uri: Url) -> anyhow::Result<Self> {
+		let (source, pre_parse_tree) = pre::parse(source)?;
+		eprintln!("Pre-parsed {uri}: {pre_parse_tree:#?}");
+
 		let (ast, source, tokens) = parse(source)?;
 		let rope = Rope::from_str(&source);
 		let scopes = parser::scopes::build(&ast);

packages/server/src/ipc/mod.rs

@@ -35,11 +35,11 @@ pub struct IpcPlugin {
 impl IpcPlugin {
 	pub fn initialize(connection: Connection, io_threads: IoThreads) -> anyhow::Result<Self> {
 		let server_caps = capabilities::define();
-		eprintln!("{server_caps:#?}");
+		// eprintln!("{server_caps:#?}");
 
 		let init_params = connection.initialize(serde_json::to_value(server_caps)?)?;
 		let init_params = serde_json::from_value::<InitializeParams>(init_params)?;
-		eprintln!("{init_params:#?}");
+		// eprintln!("{init_params:#?}");
 
 		Ok(Self {
 			connection: Mutex::new(Some(connection)),

packages/server/src/main.rs

@@ -22,6 +22,7 @@ mod documents;
 mod hover;
 mod ipc;
 mod lsp_extensions;
+mod pre;
 mod references;
 mod semantic_tokens;
 mod utils;

packages/server/src/pre.rs

@@ -0,0 +1,263 @@
+use bevy_derive::{Deref, DerefMut};
+use gramatika::{
+	ArcStr, DebugLisp, Parse, ParseStream, Span, Spanned, SpannedError, Substr, Token,
+};
+use parser::ParseStreamer;
+
+use self::{
+	chunk::{Chunk, ChunkKind},
+	dir::Directive,
+};
+
+mod chunk {
+	use core::fmt;
+
+	use gramatika::{DebugLisp, DebugLispToken, Span, Substr, Token};
+
+	#[derive(Token, gramatika::Lexer, DebugLispToken)]
+	pub enum Chunk {
+		#[pattern = r" *#if(n?def)? .+"]
+		BranchStart(Substr, Span),
+
+		#[pattern = r" *#else .+"]
+		#[pattern = r" *#else"]
+		BranchFork(Substr, Span),
+
+		#[pattern = r" *#endif .+"]
+		#[pattern = r" *#endif"]
+		BranchEnd(Substr, Span),
+
+		#[pattern = r".+"]
+		Line(Substr, Span),
+	}
+
+	impl fmt::Debug for Chunk {
+		fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+			DebugLisp::fmt(self, f, 0)
+		}
+	}
+
+	impl fmt::Display for Chunk {
+		fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+			write!(f, "{}", self.lexeme())
+		}
+	}
+}
+
+mod dir {
+	use core::fmt;
+
+	use gramatika::{DebugLisp, DebugLispToken, Span, Substr, Token};
+
+	#[derive(Token, gramatika::Lexer, DebugLispToken)]
+	pub enum Directive {
+		#[pattern = "#ifn?def"]
+		IfDef(Substr, Span),
+
+		#[pattern = "#if"]
+		If(Substr, Span),
+
+		#[pattern = "#else ifn?def"]
+		ElseIfDef(Substr, Span),
+
+		#[pattern = "#else if"]
+		ElseIf(Substr, Span),
+
+		#[pattern = "#else"]
+		Else(Substr, Span),
+
+		#[pattern = "#endif"]
+		Endif(Substr, Span),
+
+		#[discard]
+		#[pattern = r"\s*//.*"]
+		Comment(Substr, Span),
+
+		#[pattern = r".+"]
+		Expr(Substr, Span),
+	}
+
+	impl fmt::Debug for Directive {
+		fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+			DebugLisp::fmt(self, f, 0)
+		}
+	}
+
+	impl fmt::Display for Directive {
+		fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+			write!(f, "{}", self.lexeme())
+		}
+	}
+}
+
+#[derive(DebugLisp, Default, Deref, DerefMut)]
+pub struct PreParsed {
+	pub sections: Vec<Section>,
+}
+
+#[derive(DebugLisp)]
+pub enum Section {
+	Text(Text),
+	Branch(Box<Branch>),
+	Error(SpannedError),
+}
+
+pub struct Text {
+	pub text: Substr,
+	pub span: Span,
+}
+
+#[derive(DebugLisp)]
+pub struct Branch {
+	pub directive: Directive,
+	pub condition: Option<parser::expr::Expr>,
+	pub body: Vec<Section>,
+	pub else_: Option<Box<Branch>>,
+}
+
+pub fn parse<S>(input: S) -> gramatika::Result<(ArcStr, PreParsed)>
+where S: Into<ArcStr> {
+	let input = input.into();
+	let mut parser = ParseStream::<Chunk, chunk::Lexer>::from(input.clone());
+	let result = parser.parse()?;
+
+	Ok((input, result))
+}
+
+impl Parse for PreParsed {
+	type Stream = ParseStream<Chunk, chunk::Lexer>;
+
+	fn parse(input: &mut Self::Stream) -> gramatika::Result<Self> {
+		let mut result = Self::default();
+
+		while !input.is_empty() {
+			result.push(input.parse()?);
+		}
+
+		Ok(result)
+	}
+}
+
+impl Parse for Section {
+	type Stream = ParseStream<Chunk, chunk::Lexer>;
+
+	fn parse(input: &mut Self::Stream) -> gramatika::Result<Self> {
+		match input.peek().cloned() {
+			Some(Chunk::Line(_, _)) => Ok(Section::Text(input.parse()?)),
+			Some(Chunk::BranchStart(_, _)) => Ok(Section::Branch(Box::new(input.parse()?))),
+			Some(_) => {
+				let chunk = input.next().unwrap();
+				Ok(Section::Error(SpannedError {
+					message: "Unexpected input".into(),
+					source: input.source(),
+					span: Some(chunk.span()),
+				}))
+			}
+			None => Ok(Section::Error(SpannedError {
+				message: "Unexpected end of input".into(),
+				source: input.source(),
+				span: input.prev().map(|token| token.span()),
+			})),
+		}
+	}
+}
+
+impl Parse for Text {
+	type Stream = ParseStream<Chunk, chunk::Lexer>;
+
+	fn parse(input: &mut Self::Stream) -> gramatika::Result<Self> {
+		let (mut text, mut span) = input.consume_kind(ChunkKind::Line)?.as_inner();
+
+		while let Some(Chunk::Line(next_text, next_span)) = input.peek() {
+			text = parser::utils::join_substrs(&text, next_text);
+			span = span.through(*next_span);
+			input.next();
+		}
+
+		Ok(Self { text, span })
+	}
+}
+
+impl Parse for Branch {
+	type Stream = ParseStream<Chunk, chunk::Lexer>;
+
+	fn parse(input: &mut Self::Stream) -> gramatika::Result<Self> {
+		// TODO: We could use this span to adjust the spans of the directive and
+		//       condition, but is it worth the effort?
+		let (line, _line_span) = match input.next() {
+			Some(Chunk::BranchStart(line, span) | Chunk::BranchFork(line, span)) => (line, span),
+			Some(other) => {
+				return Err(SpannedError {
+					message: "Expected `#if ...`, `#ifdef ...` or `#else ...`".into(),
+					source: input.source(),
+					span: Some(other.span()),
+				});
+			}
+			None => {
+				return Err(SpannedError {
+					message: "Unexpected end of input".into(),
+					source: input.source(),
+					span: input.prev().map(|token| token.span()),
+				});
+			}
+		};
+		// eprintln!("PROCESSING BRANCH:\n   `{line}` ({line_span:?})");
+
+		let mut dir_parser = ParseStream::<Directive, dir::Lexer>::from(line.as_str());
+		let directive = dir_parser
+			.next()
+			.ok_or_else(|| unexpected_eof(dir_parser.source()))?;
+
+		let condition = if dir_parser.check_kind(dir::DirectiveKind::Expr) {
+			let expr = dir_parser.consume_kind(dir::DirectiveKind::Expr)?.lexeme();
+			let mut wgsl_parser = parser::ParseStream::from(expr.as_str());
+			Some(wgsl_parser.parse()?)
+		} else {
+			None
+		};
+
+		let mut body = vec![];
+		while !input.check_kind(ChunkKind::BranchEnd) && !input.check_kind(ChunkKind::BranchFork) {
+			body.push(input.parse()?);
+		}
+
+		let else_ = if input.check_kind(ChunkKind::BranchFork) {
+			Some(Box::new(input.parse()?))
+		} else {
+			None
+		};
+
+		if !directive.lexeme().starts_with("#else") {
+			input.consume_kind(ChunkKind::BranchEnd)?;
+		}
+
+		Ok(Self {
+			directive,
+			condition,
+			body,
+			else_,
+		})
+	}
+}
+
+fn unexpected_eof(source: ArcStr) -> SpannedError {
+	SpannedError {
+		message: "Unexpected end of input".into(),
+		source,
+		span: None,
+	}
+}
+
+impl DebugLisp for Text {
+	fn fmt(&self, f: &mut std::fmt::Formatter, indent: usize) -> std::fmt::Result {
+		writeln!(f, r#"(Text """"#)?;
+
+		for (idx, line) in self.text.lines().enumerate() {
+			writeln!(f, "{:>3} | {line}", self.span.start.line + idx + 1)?;
+		}
+
+		write!(f, r#"{}""")"#, gramatika::debug::INDENT.repeat(indent))?;
+
+		Ok(())
+	}
+}