Update text_splitter to 4000 chunk_size and 200 overlap

punitarani · punitarani · commit 2ae5e45d031d · 2023-05-14T03:44:35.000-07:00
diff --git a/rabbithole/loader.py b/rabbithole/loader.py
@@ -33,7 +33,7 @@ def load_file(file: UploadedFile) -> list[Document]:
     Supported file types: PDF
     :return: List of Document objects
     """
-    text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)
+    text_splitter = CharacterTextSplitter(chunk_size=4000, chunk_overlap=200)
 
     # Handle .docx files
     if file.name.endswith(".docx"):
