U
    h"                     @   s   d dl Z d dlmZ d dlmZ d dlmZmZmZmZm	Z	m
Z
mZ d dlZd dlmZ d dlmZmZmZ d dlmZ d dlmZ G d	d
 d
eeeZG dd deZG dd deeZdS )    N)ABC)datetime)CallableDictIteratorListLiteralOptionalUnionDocument)	BaseModelroot_validator	validatorget_from_dict_or_env)
BaseLoaderc                   @   sb   e Zd ZU dZeed< eed< dZeed< eddeedd	d
Z	e
eeef dddZdS )BaseGitHubLoaderz Load `GitHub` repository Issues.repoaccess_tokenzhttps://api.github.comgithub_api_urlT)pre)valuesreturnc                 C   s   t |dd|d< |S )z1Validate that access token exists in environment.r   ZGITHUB_PERSONAL_ACCESS_TOKENr   )clsr    r   O/tmp/pip-unpacked-wheel-9gdii04g/langchain_community/document_loaders/github.pyvalidate_environment   s      z%BaseGitHubLoader.validate_environmentr   c                 C   s   dd| j  dS )Nzapplication/vnd.github+jsonzBearer )AcceptAuthorization)r   selfr   r   r   headers    s    
zBaseGitHubLoader.headersN)__name__
__module____qualname____doc__str__annotations__r   r   r   r   propertyr#   r   r   r   r   r      s   
r   c                   @   sR  e Zd ZU dZdZeed< dZee	e
d df ed< dZee
d  ed< dZee ed	< dZee ed
< dZee ed< dZeee  ed< dZee
d  ed< dZee
d  ed< dZee ed< dZee	 ed< dZee	 ed< edddee ee dddZee dddZeedddZeedddZ eedd d!Z!dS )"GitHubIssuesLoaderz#Load issues of a GitHub repository.Tinclude_prsN)*none	milestone)openclosedallstateassigneecreator	mentionedlabels)createdupdatedcommentssort)Zascdesc	directionsincepageper_page)Zallow_reuse)vr   c                 C   s<   |r8zt |d W n" tk
r6   td| Y nX |S )Nz%Y-%m-%dT%H:%M:%SZz\Invalid value for 'since'. Expected a date string in YYYY-MM-DDTHH:MM:SSZ format. Received: )r   strptime
ValueError)r   rA   r   r   r   validate_sinceK   s    
z!GitHubIssuesLoader.validate_sincer   c                 c   s   | j }|rtj|| jd}|  | }|D ]&}| |}| jsN|jd rNq.|V  q.|j	r|j	dr| j
s| js|j	d d }qd}qdS )a  
        Get issues of a GitHub repository.

        Returns:
            A list of Documents with attributes:
                - page_content
                - metadata
                    - url
                    - title
                    - creator
                    - created_at
                    - last_update_time
                    - closed_time
                    - number of comments
                    - state
                    - labels
                    - assignee
                    - assignees
                    - milestone
                    - locked
                    - number
                    - is_pull_request
        r#   is_pull_requestnexturlN)rH   requestsgetr#   raise_for_statusjsonparse_issuer,   metadatalinksr?   r@   )r"   rH   responseZissuesissuedocr   r   r   	lazy_loadW   s(    

zGitHubIssuesLoader.lazy_load)rQ   r   c                 C   s   |d |d |d d |d |d |d dd	 |d
 D |d rL|d d nd|d rb|d d nd|d |d d|kd}|d dk	r|d nd}t ||dS )z5Create Document objects from a list of GitHub issues.Zhtml_urltitleuserlogin
created_atr:   r3   c                 S   s   g | ]}|d  qS )namer   ).0labelr   r   r   
<listcomp>   s     z2GitHubIssuesLoader.parse_issue.<locals>.<listcomp>r7   r4   Nr/   lockednumberZpull_request)rH   rT   r5   rW   r:   r3   r7   r4   r/   r\   r]   rF   body Zpage_contentrN   r   )r"   rQ   rN   contentr   r   r   rM      s    
zGitHubIssuesLoader.parse_issuec                 C   sh   | j rd| j n| j }| j| j| j| j| j|| j| j| j	| j
| jd}dd | D }d|}|S )z'Create query parameters for GitHub API.,)r/   r3   r4   r5   r6   r7   r;   r=   r>   r?   r@   c                 S   s&   g | ]\}}|d k	r| d| qS )N=r   )rY   krA   r   r   r   r[      s     z3GitHubIssuesLoader.query_params.<locals>.<listcomp>&)r7   joinr/   r3   r4   r5   r6   r;   r=   r>   r?   r@   items)r"   r7   Zquery_params_dictZquery_params_listquery_paramsr   r   r   rh      s$    
zGitHubIssuesLoader.query_paramsc                 C   s   | j  d| j d| j S )zCreate URL for GitHub API./repos/z/issues?)r   r   rh   r!   r   r   r   rH      s    zGitHubIssuesLoader.url)"r$   r%   r&   r'   r,   boolr)   r/   r
   intr   r3   r	   r4   r(   r5   r6   r7   r   r;   r=   r>   r?   r@   r   rD   r   r   rS   dictrM   r*   rh   rH   r   r   r   r   r+   (   s*   

+r+   c                   @   sh   e Zd ZU dZdZeed< eeege	f  ed< e
e dddZeedd	d
Zee dddZdS )GithubFileLoaderzLoad GitHub Filemainbranchfile_filterr   c                    sR    j  d j d j d}tj| jd}|  | d } fdd|D S )Nri   z/git/trees/z?recursive=1rE   treec                    s$   g | ]} j r  |d  r|qS )path)rp   )rY   fr!   r   r   r[      s    z3GithubFileLoader.get_file_paths.<locals>.<listcomp>)r   r   ro   rI   rJ   r#   rK   rL   )r"   base_urlrP   	all_filesr   r!   r   get_file_paths   s    

zGithubFileLoader.get_file_paths)rr   r   c                 C   sx   | j rd| j  nd}| j d| j d| | }tj|| jd}|  t| t	rt| d }t
|dS dS )Nz?ref=r_   ri   z
/contents/rE   ra   zutf-8)ro   r   r   rI   rJ   r#   rK   
isinstancerL   rl   base64	b64decodedecode)r"   rr   Zqueryparamsrt   rP   Zcontent_encodedr   r   r   get_file_content_by_path   s    z)GithubFileLoader.get_file_content_by_pathc                 c   sz   |   }|D ]h}| |d }|dkr(q|d |d | j d| j d|d  d| j d|d  	d}t||dV  qd S )Nrr   r_   sha/type)rr   r|   sourcer`   )rv   r{   r   r   ro   r   )r"   filesfilera   rN   r   r   r   rS      s    ,zGithubFileLoader.lazy_loadN)r$   r%   r&   r'   ro   r(   r)   r	   r   rj   r   r   rv   r{   r   r   rS   r   r   r   r   rm      s   
rm   )rx   abcr   r   typingr   r   r   r   r   r	   r
   rI   Zlangchain_core.documentsr   Zlangchain_core.pydantic_v1r   r   r   Zlangchain_core.utilsr   Z)langchain_community.document_loaders.baser   r   r+   rm   r   r   r   r   <module>   s   $ 