مساعد يقرأ مستنداتك محلياً

from pathlib import Path
from langchain.text_splitter import RecursiveCharacterTextSplitter


def load(folder: str):
    docs = []
    for path in Path(folder).rglob('*.md'):
        docs.append(path.read_text(encoding='utf-8'))
    return docs


def chunk(docs, size=800, overlap=120):
    splitter = RecursiveCharacterTextSplitter(
        chunk_size=size, chunk_overlap=overlap
    )
    return splitter.create_documents(docs)