U
    hN                     @   sP   d dl Z d dlmZ d dlmZmZ d dlmZ d dlm	Z	 G dd de	Z
dS )    N)Path)ListUnion)Document)
BaseLoaderc                   @   s8   e Zd ZdZeeef dddZee	 dddZ
dS )	CoNLLULoaderzLoad `CoNLL-U` files.	file_pathc                 C   s
   || _ dS )zInitialize with a file path.Nr   )selfr	    r   O/tmp/pip-unpacked-wheel-9gdii04g/langchain_community/document_loaders/conllu.py__init__   s    zCoNLLULoader.__init__)returnc              	   C   s   t | jdd&}ttj|dd}dd |D }W 5 Q R X d}t|D ]B\}}|d d	ksj|t|d
 krx||d
 7 }qF||d
 d 7 }qFdt| ji}t||dgS )zLoad from a file path.utf8)encoding	)	delimiterc                 S   s   g | ]}t |d kr|qS )   )len).0liner   r   r   
<listcomp>   s      z%CoNLLULoader.load.<locals>.<listcomp> 	   zSpaceAfter=Nor    source)Zpage_contentmetadata)	openr	   listcsvreader	enumerater   strr   )r
   fZtsvlinestextir   r   r   r   r   load   s    zCoNLLULoader.loadN)__name__
__module____qualname____doc__r   r"   r   r   r   r   r'   r   r   r   r   r   
   s   r   )r   pathlibr   typingr   r   Zlangchain_core.documentsr   Z)langchain_community.document_loaders.baser   r   r   r   r   r   <module>   s
   