Fix indexing of whitespace only

onyx-dot-app · Jan 6, 2024 · 3098365 · 3098365 · vercel · Jan 6, 2024
1 parent 6b6b3da
commit 3098365
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/backend/danswer/indexing/chunker.py b/backend/danswer/indexing/chunker.py
@@ -137,7 +137,8 @@ def chunk_document(
             chunk_text = section_text
 
     # Once we hit the end, if we're still in the process of building a chunk, add what we have
-    if chunk_text:
+    # NOTE: if it's just whitespace, ignore it.
+    if chunk_text.strip():
         chunks.append(
             DocAwareChunk(
                 source_document=document,