U hw#@sddlmZddlZddlZddlmZddlmZmZm Z m Z m Z ddl m Z ddlmZertddlZddlmZdd d d d ZGd ddeZdddddZGdddeZeZdS)) annotationsN)Path) TYPE_CHECKINGDictListOptionalUnionDocument) BaseLoader) EntityLikedictstr)rowreturncCs.|d}|d}|d}|d|d|dS)zBCombine message information in a readable format ready to be used.datefromtextz on z:  )rrZsenderrrrQ/tmp/pip-unpacked-wheel-9gdii04g/langchain_community/document_loaders/telegram.pyconcatenate_rowssrc@s,eZdZdZddddZdddd Zd S) TelegramChatFileLoaderzLoad from `Telegram chat` dump.zUnion[str, Path])pathcCs ||_dS)zInitialize with a path.N) file_path)selfrrrr__init__szTelegramChatFileLoader.__init__List[Document]rc Cs^t|j}t|dd}t|}W5QRXddd|dD}dt|i}t||dgS) Load documents.utf8encodingcss0|](}|ddkrt|dtrt|VqdS)typemessagerN) isinstancerr).0r%rrr 's z.TelegramChatFileLoader.load..messagessource page_contentmetadata)rropenjsonloadjoinrr )rpfdrr-rrrr0 s   zTelegramChatFileLoader.loadN)__name__ __module__ __qualname____doc__rr0rrrrrsrzUnion[str, List[str]]r)rrc Csddlm}|ddddddd d d gd d }t|tr:|g}dd|D}t|D]\}}|d|jd<qPg}|D]d}||j}t|D]J\}}t||jd|dd}|jdd|jd|jd<| |qqp|S)zIConvert a string or list of strings to a list of Documents with metadata.r)RecursiveCharacterTextSplitteri r .!?, r#) chunk_size separatorsZ chunk_overlapcSsg|]}t|dqS))r,r )r'pagerrr >sz text_to_docs..rC)rCchunkr+-rFr*) Zlangchain_text_splittersr9r&r enumerater-Z split_textr,r append) rr9Z text_splitterZ page_docsidocZ doc_chunkschunksrFrrr text_to_docs1s,    rMc@sfeZdZdZdddddddd d Zd d d dZdddddZddddddZdd ddZdS)TelegramChatApiLoaderz)Load `Telegram` chat json directory dump.Ntelegram_data.jsonzOptional[EntityLike]z Optional[int]z Optional[str]r chat_entityapi_idapi_hashusernamercCs"||_||_||_||_||_dS)aIInitialize with API parameters. Args: chat_entity: The chat entity to fetch data from. api_id: The API ID. api_hash: The API hash. username: The username. file_path: The file path to save the data to. Defaults to "telegram_data.json". NrP)rrQrRrSrTrrrrrVs zTelegramChatApiLoader.__init__Nonerc sddlm}g}||j|j|j4IdH`}||j2zJ3dHW}|jdk }|rZ|jjnd}| |j |j |j |j||dq86W5QIdHRXt|jddd}tj||dd d W5QRXdS) z8Fetch data from Telegram API and save it as a JSON file.r)TelegramClientN) sender_idrr message.idis_reply reply_to_idwzutf-8r!F) ensure_asciiindent)Z telethon.syncrVrTrRrSZ iter_messagesrQZreply_toZreply_to_msg_idrIrWrr isoformatidr.rr/dump)rrVdataclientr%rYrZr3rrrfetch_data_from_telegramns"   z.TelegramChatApiLoader.fetch_data_from_telegram pd.DataFramer )rbrcshddddfdd ||d}||djdgd dtd<fd d |d D}|S) a Create a dictionary of message threads from the given data. Args: data (pd.DataFrame): A DataFrame containing the conversation data with columns: - message.sender_id - text - date - message.id - is_reply - reply_to_id Returns: dict: A dictionary where the key is the parent message ID and the value is a list of message IDs in ascending order. intrez List[int]) parent_id reply_datarcs>||d|kd}g}|D]}||g||7}q |S)a^ Recursively find all replies to a given parent message ID. Args: parent_id (int): The parent message ID. reply_data (pd.DataFrame): A DataFrame containing reply messages. Returns: list: A list of message IDs that are replies to the parent message ID. rZrX)tolist)rgrhZdirect_repliesZ all_repliesZreply_id) find_repliesrrrjs z@TelegramChatApiLoader._get_message_threads..find_repliesrYrZ)Zsubsetcsi|]}||g|qSrr)r'rgrjZreply_messagesrr sz>TelegramChatApiLoader._get_message_threads..rX)ZdropnaZastyperf)rrbZparent_messagesmessage_threadsrrkr_get_message_threadss z*TelegramChatApiLoader._get_message_threadszDict[int, List[int]])rmrbrcCs`d}|D]J\}}||d|jddd}dd|D}|d|d 7}q |S) aw Combine the message texts for each parent message ID based on the list of message threads. Args: message_threads (dict): A dictionary where the key is the parent message ID and the value is a list of message IDs in ascending order. data (pd.DataFrame): A DataFrame containing the conversation data: - message.sender_id - text - date - message.id - is_reply - reply_to_id Returns: str: A combined string of message texts sorted by date. r#rXr)ZbyrcSsg|] }t|qSr)r)r'elemrrrrDsz@TelegramChatApiLoader._combine_message_texts..r?z. )itemsisinZ sort_valuesrir1strip)rrmrbZ combined_textrgZ message_idsZ message_textsrrr_combine_message_textssz,TelegramChatApiLoader._combine_message_textsrc Cs|jdk rJz"ddl}|t|Wntk rHtdYnXt|j}t |dd}t |}W5QRXz ddl }Wntk rtdYnX| |}||}||}|||} t| S)rNrzy`nest_asyncio` package not found. please install with `pip install nest_asyncio` r r!zf`pandas` package not found. please install with `pip install pandas` )rQ nest_asyncioapplyasynciorunrd ImportErrorrrr.r/r0pandasZjson_normalizeZ DataFramernrsrM) rrtr2r3r4pdZnormalized_messagesZdfrmZcombined_textsrrrr0s.         zTelegramChatApiLoader.load)NNNNrO) r5r6r7r8rrdrnrsr0rrrrrNSs;&rN) __future__rrvr/pathlibrtypingrrrrrZlangchain_core.documentsr Z)langchain_community.document_loaders.baser ryrzZtelethon.hintsr rrrMrNZTelegramChatLoaderrrrrs     ":