U h(<@sddlmZddlZddlmZmZmZmZmZm Z ddl m Z ddl m Z ddlmZGdddeZGd d d ZGd d d eZGd ddeZGdddZdS)) annotationsN)AnyDictListTuple TypedDictUnionDocument)Language)RecursiveCharacterTextSplittercs(eZdZdZdddfdd ZZS)MarkdownTextSplitterz=Attempts to split the text along Markdown-formatted headings.rNone)kwargsreturnc s&|tj}tjfd|i|dS)z"Initialize a MarkdownTextSplitter. separatorsN)Zget_separators_for_languager ZMARKDOWNsuper__init__)selfrr __class__E/tmp/pip-unpacked-wheel-a648t6hw/langchain_text_splitters/markdown.pyrs zMarkdownTextSplitter.__init__)__name__ __module__ __qualname____doc__r __classcell__rrrrr sr c@sDeZdZdZdddddddZd d d d d Zdd dddZdS)MarkdownHeaderTextSplitterz4Splitting markdown files based on specified headers.FTzList[Tuple[str, str]]boolheaders_to_split_onreturn_each_line strip_headerscCs$||_t|dddd|_||_dS)a Create a new MarkdownHeaderTextSplitter. Args: headers_to_split_on: Headers we want to track return_each_line: Return each line w/ associated headers strip_headers: Strip split headers from the content of the chunk cSs t|dS)Nr)len)splitrrr*z5MarkdownHeaderTextSplitter.__init__..T)keyreverseN)r"sortedr!r#rr!r"r#rrrrsz#MarkdownHeaderTextSplitter.__init__zList[LineType]List[Document])linesrcCsg}|D]}|rB|dd|dkrB|ddd|d7<q|r|dd|dkrt|ddt|dkr|ddddddkr|js|ddd|d7<|d|dd<q||qdd |DS) zCombine lines with common metadata into chunks Args: lines: Line of text / associated header metadata metadatacontentz  r#cSs g|]}t|d|ddqSr0r/ page_contentr/r .0chunkrrr TszHMarkdownHeaderTextSplitter.aggregate_lines_to_chunks..)r$r%r#append)rr-Zaggregated_chunkslinerrraggregate_lines_to_chunks/s. z4MarkdownHeaderTextSplitter.aggregate_lines_to_chunksstrtextrcCs&|d}g}g}i}g}i}d}d} |D]} | } dttj| } |s| drp| ddkrpd}d} q| drd}d} n| | rd}d} |r|| q*|j D]\} } | | rt | t | ks| t | dkr| d k r`| d }|r.|d d |kr.| }|d |kr| |d q|| | t | d d}|||d|| <|r|d|| d| |js|| qq| r|| n(|r|d|| d| | }q*|r|d||d|js||Sdd|DSd S)zASplit markdown file Args: text: Markdown filer1Fz```Tz~~~ Nr2r.levelname)rCrDdatarE)r0r/cSs g|]}t|d|ddqSr3r r6rrrr9sz9MarkdownHeaderTextSplitter.split_text..)r%stripjoinfilterr= isprintable startswithcountr:r!r$popcopyclearr#r"r<)rr?r-Zlines_with_metadataZcurrent_contentZcurrent_metadataZ header_stackZinitial_metadataZ in_code_blockZ opening_fencer;Z stripped_lineseprDZcurrent_header_levelZ popped_headerheaderrrr split_textYs               z%MarkdownHeaderTextSplitter.split_textN)FT)rrrrrr<rQrrrrrs *rc@s"eZdZUdZded<ded<dS)LineTypezLine type as typed dict.zDict[str, str]r/r=r0Nrrrr__annotations__rrrrrRs rRc@s*eZdZUdZded<ded<ded<dS) HeaderTypezHeader type as typed dict.intrCr=rDrENrSrrrrrUs rUc@seZdZdZdddddddZd*d d d dddZdddddZddddddZddddddZddd d!Z dd"d#d$d%Z dd"d#d&d'Z dd"d#d(d)Z d S)+&ExperimentalMarkdownSyntaxTextSplittera An experimental text splitter for handling Markdown syntax. This splitter aims to retain the exact whitespace of the original text while extracting structured metadata, such as headers. It is a re-implementation of the MarkdownHeaderTextSplitter with notable changes to the approach and additional features. Key Features: - Retains the original whitespace and formatting of the Markdown text. - Extracts headers, code blocks, and horizontal rules as metadata. - Splits out code blocks and includes the language in the "Code" metadata key. - Splits text on horizontal rules (`---`) as well. - Defaults to sensible splitting behavior, which can be overridden using the `headers_to_split_on` parameter. Parameters: ---------- headers_to_split_on : List[Tuple[str, str]], optional Headers to split on, defaulting to common Markdown headers if not specified. return_each_line : bool, optional When set to True, returns each line as a separate chunk. Default is False. Usage example: -------------- >>> headers_to_split_on = [ >>> ("#", "Header 1"), >>> ("##", "Header 2"), >>> ] >>> splitter = ExperimentalMarkdownSyntaxTextSplitter( >>> headers_to_split_on=headers_to_split_on >>> ) >>> chunks = splitter.split(text) >>> for chunk in chunks: >>> print(chunk) This class is currently experimental and subject to change based on feedback and further development. zHeader 1zHeader 2zHeader 3zHeader 4zHeader 5zHeader 6)r2z##z###z####z#####z######NFTz"Union[List[Tuple[str, str]], None]rr cCs@g|_tdd|_g|_||_|r.t||_n|j|_||_dS)Nr@r5) chunksr current_chunkcurrent_header_stackr#dictsplittable_headersDEFAULT_HEADER_KEYSr"r+rrrrs  z/ExperimentalMarkdownSyntaxTextSplitter.__init__r=r,r>c Cs|jdd}|r|d}||}||}||}|r||jsZ|jj|7_t | d}| d}| ||q |r|| |||j_| d|jj d<|q |r|q |jj|7_q ||jrdd|jDS|jS) NT)keependsrrAZCodecSs6g|].}|jD]}|r|st||jdqqS)r4)r5 splitlinesisspacer r/)r7r8r;rrrr9Is  zEExperimentalMarkdownSyntaxTextSplitter.split_text..)rarL _match_header _match_code _match_horz_complete_chunk_docr#rZr5r$group_resolve_header_stack_resolve_code_chunkr/r"rY) rr? raw_linesraw_lineZ header_matchZ code_matchZ horz_match header_depth header_textrrrrQ&s<        z1ExperimentalMarkdownSyntaxTextSplitter.split_textrVr)rlrmrcCs\t|jD]<\}\}}||kr ||f|j|<|jd|d|_dSq |j||fdS)NrA) enumerater[r:)rrlrmidepth_rrrrhQs z.)z^```(.*)z^~~~(.*)css|]}|r|VqdSNrr7rvrrr xszEExperimentalMarkdownSyntaxTextSplitter._match_code..nextrr;matchesrryrrdvsz2ExperimentalMarkdownSyntaxTextSplitter._match_codecs&fdddD}tdd|DdS)Ncsg|]}t|qSrrwrxryrrr9{szFExperimentalMarkdownSyntaxTextSplitter._match_horz..)z ^\*\*\*+\nz^---+\nz^___+\ncss|]}|r|VqdSrzrr{rrrr|~szEExperimentalMarkdownSyntaxTextSplitter._match_horz..r}rrryrrezs z2ExperimentalMarkdownSyntaxTextSplitter._match_horz)NFT) rrrrr^rrQrhrirfrcrdrerrrrrWs&) +  rW) __future__rrutypingrrrrrrZlangchain_core.documentsr Zlangchain_text_splitters.baser Z"langchain_text_splitters.characterr r rrRrUrWrrrrs      @