U h^-@sddlmZddlZddlZddlmZmZddlmZm Z m Z m Z m Z m Z mZmZddlZddlmZddlmZGdddeZGd d d ZGd d d ZdS) ) annotationsN)BytesIOStringIO)AnyDictIterableListOptionalTuple TypedDictcastDocument)RecursiveCharacterTextSplitterc@s2eZdZUdZded<ded<ded<ded<dS) ElementTypezElement type as typed dict.strurlxpathcontentzDict[str, str]metadataN)__name__ __module__ __qualname____doc____annotations__rrA/tmp/pip-unpacked-wheel-a648t6hw/langchain_text_splitters/html.pyrs rc@sdeZdZdZddddddZdd d d d Zd dd dddZd d dddZdd dddZdS)HTMLHeaderTextSplitterzU Splitting HTML files based on specified headers. Requires lxml package. FList[Tuple[str, str]]bool)headers_to_split_onreturn_each_elementcCs||_t||_dS)ayCreate a new HTMLHeaderTextSplitter. Args: headers_to_split_on: list of tuples of headers we want to track mapped to (arbitrary) keys for metadata. Allowed header values: h1, h2, h3, h4, h5, h6 e.g. [("h1", "Header 1"), ("h2", "Header 2)]. return_each_element: Return each element w/ associated headers. N)r!sortedr )selfr r!rrr__init__szHTMLHeaderTextSplitter.__init__zList[ElementType]List[Document])elementsreturncCs\g}|D]D}|rB|dd|dkrB|ddd|d7<q||qdd|DS)zCombine elements with common metadata into chunks Args: elements: HTML element content with associated identifying info and metadata rrz cSs g|]}t|d|ddqSrr page_contentrr .0chunkrrr EszGHTMLHeaderTextSplitter.aggregate_elements_to_chunks..)append)r#r&Zaggregated_chunkselementrrraggregate_elements_to_chunks.s z3HTMLHeaderTextSplitter.aggregate_elements_to_chunksrr)rkwargsr'cKstj|f|}|t|jS)zSplit HTML from web URL Args: url: web URL **kwargs: Arbitrary additional keyword arguments. These are usually passed to the fetch url content request. )requestsgetsplit_text_from_filerr)r#rr3rrrrsplit_text_from_urlJsz*HTMLHeaderTextSplitter.split_text_from_urltextr'cCs|t|SzJSplit HTML text string Args: text: HTML text r6rr#r:rrr split_textUsz!HTMLHeaderTextSplitter.split_textfiler'csjzddlm}Wn,tk r<}ztd|W5d}~XYnX|jdd}|||}ttjd}||}| |}||} | t | } dd |j Dt |j d d i} g} | d | D]} | d s| dr| t|ddd | d| Dddd | d| Dfddtfdd| d| Ddq|jsX|| Sdd | DSdS)CSplit HTML file Args: file: HTML file retree>Unable to import lxml, please install with `pip install lxml`.Nzutf-8)encodingz!xsl/html_chunks_with_headers.xsltcSsg|] }|dqS)rr)r-headerrrrr/xsz?HTMLHeaderTextSplitter.split_text_from_file..hzhttp://www.w3.org/1999/xhtmlz*//*z*[@class='headers']z*[@class='chunk']cSsg|]}|jpdqSrHr:r-noderrrr/sz*[@class='xpath']cSsg|]}|jpdqSrIrJrKrrrr/scsi|]}|j|jpdqSrI)tagr:rK)header_mappingrr sz?HTMLHeaderTextSplitter.split_text_from_file..cs |jkS)N)rM)x) header_filterrrz=HTMLHeaderTextSplitter.split_text_from_file..z*[@class='headers']/*)rrrrcSs g|]}t|d|ddqSr)r r,rrrr/s)lxmlrC ImportError HTMLParserparsepathlibPath__file__parentXSLT fromstringrr dictfindallr0rjoinfilterr!r2)r#r@rCeparsertree xslt_path xslt_tree transformresultZ result_domZns_mapr&r1r)rQrNrr6]sb           z+HTMLHeaderTextSplitter.split_text_from_fileN)F) rrrrr$r2r8r>r6rrrrrs rc@seZdZdZd"ddddddd Zd d d d dZdd dddZd#ddd dddZdddddZdddddZ dd dd d!Z dS)$HTMLSectionSplitterz` Splitting HTML files based on specified tag and font sizes. Requires lxml package. Nrz Optional[str]rNone)r rer3r'cKsDt||_|dkr*ttjd|_nt||_||_dS)aCreate a new HTMLSectionSplitter. Args: headers_to_split_on: list of tuples of headers we want to track mapped to (arbitrary) keys for metadata. Allowed header values: h1, h2, h3, h4, h5, h6 e.g. [("h1", "Header 1"), ("h2", "Header 2"]. xslt_path: path to xslt file for document transformation. Uses a default if not passed. Needed for html contents that using different format and layouts. Nzxsl/converting_to_header.xslt) r^r rXrYrZr[absoluterer3)r#r rer3rrrr$s   zHTMLSectionSplitter.__init__zIterable[Document]r%) documentsr'cCsPgg}}|D]}||j||jq|j||d}tf|j}||S)zSplit documents.) metadatas)r0r+rcreate_documentsrr3split_documents)r#rltextsrmdocresultsZ text_splitterrrrros   z#HTMLSectionSplitter.split_documentsrr9cCs|t|Sr;r<r=rrrr>szHTMLSectionSplitter.split_textz List[str]zOptional[List[dict]])rprmr'c Cs|pigt|}g}t|D]t\}}||D]`}t||}|jD] } |j| dkrL|d|j| <qL||j}t|j|d} | | q0q|S)z&Create documents from a list of texts.#TITLE#ZTitler*) len enumerater>copydeepcopyrkeysrr+r0) r#rprmZ _metadatasrlir:r.rkeyZnew_docrrrrns z$HTMLSectionSplitter.create_documentszList[Dict[str, Optional[str]]])html_docr'c Cszddlm}m}Wn,tk r@}ztd|W5d}~XYnX||d}t|j}g}|dg|}t|D]\}} | } |dkrd} d} g} n| j } | j } g} | j D]<}|dt |kr|||dkrqt|tr| |qd |  }|d krv|| || d qv|S) Nr) BeautifulSoup PageElementzzUnable to import BeautifulSoup/PageElement, please install with `pip install bs4`.z html.parserbodyrsh1 rH)rFrtag_name)Zbs4r|r}rUlistr rxZfind_allrur:stripnameZ next_elementsrt isinstancerr0r`)r#r{r|r}rbZsoupheaderssectionsryrFZheader_elementZcurrent_headerZcurrent_header_tagZsection_contentr1rrrrsplit_html_by_headerssF      z)HTMLSectionSplitter.split_html_by_headers) html_contentr'c Cs|jdkr|Szddlm}Wn,tk rJ}ztd|W5d}~XYnX|}|t||}||j}||}||}t|S)NrrBrD) rerTrCrUrVrWrr\r) r#rrCrbrcrdrfrgrhrrrconvert_possible_tags_to_headers    z3HTMLSectionSplitter.convert_possible_tags_to_headerr?cs.|}|}|}fdd|DS)rAcs8g|]0}ttt|djt|d|didqS)rrrF)r)rr rr )r-sectionr#rrr/7s  z.)getvaluerr)r#r@Z file_contentrrrrr6-s    z(HTMLSectionSplitter.split_text_from_file)N)N) rrrrr$ror>rnrrr6rrrrris  ,ri) __future__rrvrXiorrtypingrrrrr r r r r4Zlangchain_core.documentsrZ"langchain_text_splitters.characterrrrrirrrrs (