U hX$@sdZddlZddlZddlZddlmZddlmZmZm Z m Z ddl m Z ddl mZddlmZmZmZmZmZmZmZmZmZmZmZmZeeZGdd d eZdS) z:Pebblo's safe dataloader is a wrapper for document loadersN)version)DictIteratorListOptional)Document) BaseLoader) BATCH_SIZE_BYTESPLUGIN_VERSIONApp FrameworkIndexedDocumentPebbloLoaderAPIWrappergenerate_size_based_batches get_full_pathget_loader_full_pathget_loader_type get_runtimeget_source_sizec @seZdZUdZdZeed<d"ddeeeee eee eedd d Z e e d d d Z dd ddZee d ddZedd ddZed ddZe ed ddZee e dddZe e d ddZe ee dddZeddd d!ZdS)#PebbloSafeLoaderzkPebblo Safe Loader class is a wrapper around document loaders enabling the data to be scrutinized. F_discover_sentNlocal)classifier_location)langchain_loadernameowner descriptionapi_key load_semanticclassifier_urlrc Cs|rt|tstd||_tt|_||_tj dp>||_ ||_ ||_ t|j|_g|_g|_tt|jdddd} t| |_t|j|_t|_| |j|jd|jdkrdt|jini|_||_t|||d |_|j|jdS) NzMust specify a valid name.ZPEBBLO_LOAD_SEMANTIC.'r)loader source_path source_typesource_path_size)rrr ) isinstancestr NameErrorapp_nameuuidZuuid4load_idr$osenvirongetrrrrr%docs docs_with_idtypesplitrr&rr'r batch_sizeloader_details_get_app_detailsappr pb_clientZsend_loader_discover) selfrrrrrrr rZ loader_namer;O/tmp/pip-unpacked-wheel-9gdii04g/langchain_community/document_loaders/pebblo.py__init__%s<  "   zPebbloSafeLoader.__init__)returncCs|j|_||jS)zxLoad Documents. Returns: list: Documents fetched from load method of the wrapped `loader`. )r$loadr1classify_in_batches)r:r;r;r<r?Ts zPebbloSafeLoader.loadc Cst|j|j}g}t|}t|D]l\}}||dk}||_||_|jj|j|j |j |d}| ||j r|| |}n|}||q"||_dS)z Classify documents in batches. This is to avoid API timeouts when sending large number of documents. Batches are generated based on the page_content size. )Z loading_endN)rr1r5len enumerate _index_docsr2r9classify_documentsr8r6_add_pebblo_specific_metadatar_add_semantic_to_docs _unindex_docsextend) r:ZbatchesZprocessed_docsZ total_batchesibatchZ is_last_batchclassified_docsZbatch_processed_docsr;r;r<r@_s,     z$PebbloSafeLoader.classify_in_batchesc csz|j}WnFtk rT}z(|jjjd}t|t||W5d}~XYnXz t|}Wntk rg|_ YqYnXt |f|_ | |_ |j |j |j|j}|||jr|||_ n ||_ |j dVqVdS)zLoad documents in lazy fashion. Raises: NotImplementedError: raised when lazy_load id not implemented within wrapped loader. Yields: list: Documents from loader's lazy loading. z does not implement lazy_load()Nr)r$ lazy_loadNotImplementedError __class____name__loggererrornext StopIterationr1listrDr2r9rEr8r6rFrrGrH)r:Z doc_iteratorexcerr_strdocclassified_docr;r;r<rMs.        zPebbloSafeLoader.lazy_loadcCs d|_dS)NT)r)clsr;r;r<set_discover_sentsz"PebbloSafeLoader.set_discover_sentc Cs:t\}}t|j|j|j|j||ttdtddd}|S)z\Fetch app details. Internal method. Returns: App: App details. Zlangchain_community)rr)rrrr-runtime frameworkZplugin_versionclient_version) rr r+rrr-r r r)r:r]r\r8r;r;r<r7s  z!PebbloSafeLoader._get_app_detailscCsddt|jD}|S)z Indexes the documents and returns a list of IndexedDocument objects. Returns: List[IndexedDocument]: A list of IndexedDocument objects with unique IDs. cSs*g|]"\}}tfdt|i|qS)pb_id)r r)dict.0rJrXr;r;r< sz0PebbloSafeLoader._index_docs..)rCr1)r:r2r;r;r<rDszPebbloSafeLoader._index_docs)rLr>cCsVdd|jD}|D]&}|d}||kr||||qdd|D}|S)aF Adds semantic metadata to the given list of documents. Args: classified_docs (Dict): A dictionary of dictionaries containing the classified documents with pb_id as key. Returns: List[Document]: A list of Document objects with added semantic metadata. cSs i|]}|jt|j|jdqS) page_contentmetadata)r_rrerfrbrXr;r;r< sz:PebbloSafeLoader._add_semantic_to_docs..r_cSsg|]}|qSr;r;rgr;r;r<rcsz:PebbloSafeLoader._add_semantic_to_docs..)r2valuesr0_add_semantic_to_doc)r:rLZ indexed_docsrYZdoc_idZsemantic_metadata_docsr;r;r<rGs   z&PebbloSafeLoader._add_semantic_to_docscCsddt|jD}|S)z Converts a list of IndexedDocument objects to a list of Document objects. Returns: List[Document]: A list of Document objects. cSs g|]\}}t|j|jdqSrd)rrerfrar;r;r<rcsz2PebbloSafeLoader._unindex_docs..)rCr2)r:r1r;r;r<rHszPebbloSafeLoader._unindex_docs)rXrYr>cCs8t|di|jd<t|di|jd<|S)a4 Adds semantic metadata to the given document in-place. Args: doc (Document): A Document object. classified_doc (dict): A dictionary containing the classified document. Returns: Document: The Document object with added semantic metadata. entitiesZpebblo_semantic_entitiesZtopicsZpebblo_semantic_topics)rUr0keysrf)r:rXrYr;r;r<rjs   z%PebbloSafeLoader._add_semantic_to_docc Cst|jD]h}|j}|jjjdkr6t|d|j|d<nt|d|d|j|d<||jidd|d<qdS)z*Add Pebblo specific metadata to documents.ZSharePointLoadersource full_pathZ pb_checksumN) r2rfr$rOrPrr0r%r_)r:rLrXZ doc_metadatar;r;r<rFs    z.PebbloSafeLoader._add_pebblo_specific_metadata)rrNFN)rP __module__ __qualname____doc__rbool__annotations__rr)rr=rrr?r@rrM classmethodr[r r7r rDrrGrHr`rjrFr;r;r;r<rs<    /  "  r) rqloggingr.r,importlib.metadatartypingrrrrZlangchain_core.documentsrZ)langchain_community.document_loaders.baserZ$langchain_community.utilities.pebblor r r r r rrrrrrr getLoggerrPrQrr;r;r;r<s   8