
    	h                         d Z ddlZddlZddlZddlZddlmZ ddlmZm	Z	 ddl
mZ ddlmZ ddlmZ ddlmZ ddlmZ ded	efd
Zdej        dedee         fdZeddddddeej                 dedee         dee         dededej        fd            Zdededeej                 fdZ  e	d          Z! e	d          Z"dee!         dee"         dee#e!e"f                  fdZ$eddd ded!ed"ee%         dee         ddf
d#            Z&dS )$zfBeta utility functions to assist in common eval workflows.

These functions may change in the future.
    N)Sequence)OptionalTypeVar)
evaluation)	warn_beta)Clientrun_dictid_mapc                 4   | d         }|                                 D ]5\  }}|                    t          |          t          |                    }6|| d<   |                     d          r|| d                  | d<   |                     d          si | d<   | S )a  Convert the IDs in the run dictionary using the provided ID map.

    Parameters:
    - run_dict (dict): The dictionary representing a run.
    - id_map (dict): The dictionary mapping old IDs to new IDs.

    Returns:
    - dict: The updated run dictionary.
    dotted_orderparent_run_idextra)itemsreplacestrget)r	   r
   dokvs        b/var/www/html/web-builder-api.evdpl.com/venv/lib/python3.11/site-packages/langsmith/beta/_evals.py_convert_idsr      s     
.	!B ( (1ZZAA''!H^||O$$ F$*8O+D$E!<<   O    rootrun_to_example_mapreturnc                    | g}t          j                    }| j        |ig }|r|                                }|                    h d          }                    |d         t          j                              |d         <   |d                  |d<   |d                  |d<   |j        r|                    |j                   |                    |           |fd|D             }|| j	                 |d         d<   |S )a&  Convert the root run and its child runs to a list of dictionaries.

    Parameters:
    - root (ls_schemas.Run): The root run to convert.
    - run_to_example_map (dict): The dictionary mapping run IDs to example IDs.

    Returns:
    - List[dict]: The list of converted run dictionaries.
    >   
session_idchild_run_idsparent_run_ids)excludeidtrace_idc                 0    g | ]}t          |          S  )r   ).0rr
   s     r   
<listcomp>z%_convert_root_run.<locals>.<listcomp>A   s#    777!l1f%%777r   r   reference_example_id)
uuiduuid4r"   popdictr   
child_runsextendappendr!   )	r   r   runs_r"   resultssrcsrc_dictresultr
   s	           @r   _convert_root_runr5   *   s    FEz||HmX&FG
 !iikk88$U$U$U8VV!'HTNDJLL!I!Ix~/%hz&:;> 	)LL(((x     ! 8777w777F(:47(CF1I$%Mr   F)test_project_nameclientload_child_runsinclude_outputsrunsdataset_namer6   r7   r8   r9   c                   | st          d|            pt          j                                        |          }|rd | D             nd}                    d | D             |d | D             |j                   s| }nfd| D             }|p"d	t          j                    j        dd
          }t          
                    |                    }	d |	D             |	d         j        r|	d         j        n|	d         j        }
fd|D             }                    ||j        d|
                                d          }|D ]d}|d         |d         z
  }t          j                            t          j        j                  |d<   |d         |z   |d<    j        di |d|i e                    |j                  }|S )a  Convert the following runs to a dataset + test.

    This makes it easy to sample prod runs into a new regression testing
    workflow and compare against a candidate system.

    Internally, this function does the following:
        1. Create a dataset from the provided production run inputs.
        2. Create a new test project.
        3. Clone the production runs and re-upload against the dataset.

    Parameters:
    - runs (Sequence[ls_schemas.Run]): A sequence of runs to be executed as a test.
    - dataset_name (str): The name of the dataset to associate with the test runs.
    - client (Optional[Client]): An optional LangSmith client instance. If not provided,
        a new client will be created.
    - load_child_runs (bool): Whether to load child runs when copying runs.
        Defaults to False.

    Returns:
    - ls_schemas.TracerSession: The project containing the cloned runs.

    Examples:
    --------
    .. code-block:: python

        import langsmith
        import random

        client = langsmith.Client()

        # Randomly sample 100 runs from a prod project
        runs = list(client.list_runs(project_name="My Project", execution_order=1))
        sampled_runs = random.sample(runs, min(len(runs), 100))

        runs_as_test(runs, dataset_name="Random Runs")

        # Select runs named "extractor" whose root traces received good feedback
        runs = client.list_runs(
            project_name="<your_project>",
            filter='eq(name, "extractor")',
            trace_filter='and(eq(feedback_key, "user_score"), eq(feedback_score, 1))',
        )
        runs_as_test(runs, dataset_name="Extraction Good")
    z1Expected a non-empty sequence of runs. Received: )r;   c                     g | ]	}|j         
S r$   )outputsr%   r&   s     r   r'   z(convert_runs_to_test.<locals>.<listcomp>   s    '''Qqy'''r   Nc                     g | ]	}|j         
S r$   )inputsr?   s     r   r'   z(convert_runs_to_test.<locals>.<listcomp>   s    '''Q'''r   c                     g | ]	}|j         
S r$   )r!   r?   s     r   r'   z(convert_runs_to_test.<locals>.<listcomp>   s    ++++++r   )rA   r>   source_run_ids
dataset_idc                 H    g | ]}                     |j                   S ))r8   )read_runr!   )r%   r&   r7   r8   s     r   r'   z(convert_runs_to_test.<locals>.<listcomp>   s8     
 
 
GHFOOAD/OBB
 
 
r   zprod-baseline-   c                 (    i | ]}|j         |j        S r$   )source_run_idr!   )r%   es     r   
<dictcomp>z(convert_runs_to_test.<locals>.<dictcomp>   s    BBBA!/14BBBr   r   c                 :    g | ]}t          |          D ]}|S r$   )r5   )r%   root_runr	   r   s      r   r'   z(convert_runs_to_test.<locals>.<listcomp>   sK       )(4FGG   	   r   zprod-baseline)whichdataset_version)project_namereference_dataset_idmetadataend_time
start_time)tzrP   r$   )
ValueErrorrtget_cached_clientcreate_datasetcreate_examplesr!   r)   r*   hexlistlist_examplesmodified_at
created_atcreate_project	isoformatdatetimenowtimezoneutc
create_runupdate_project)r:   r;   r6   r7   r8   r9   dsr>   runs_to_copyexamplesrO   	to_createprojectnew_runlatency_r   s      ``           @r   convert_runs_to_testrp   F   s   l  YWtWWXXX-r+--F			L		9	9B+:D''$''''G
''$'''++d+++5	      

 
 
 
 
LP
 
 
 *T-Tdjll>NrPQr>R-T-TF((l(CCDDHBBBBB#+A;#:V@V    $  I ##&U$.88::
 
 $  G  E E*%(== ( 1 5 59J9N 5 O O%l3g=
DDGDD2CDDDDD
	 	A Nr   rP   c                 r   |                     |           }t          j        t                    }g }i }|D ]I}|j        !||j                                     |           n|                    |           |||j        <   J|                                D ]"\  }}t          |d           ||         _	        #|S )N)rP   c                     | j         S N)r   )r&   s    r   <lambda>z%_load_nested_traces.<locals>.<lambda>   s    q~ r   )key)
	list_runscollectionsdefaultdictr\   r   r/   r!   r   sortedr-   )	rP   r7   r:   treemapr1   all_runsrunrun_idr-   s	            r   _load_nested_tracesr~      s    66D%%  GH  (C%&--c2222NN3%mmoo W W
&,Z=U=U&V&V&V##Nr   TUlist1list2c                 F    t          t          j        | |                    S rs   )r\   	itertoolsproduct)r   r   s     r   _outer_productr      s    	!%//000r   
   )max_concurrencyr7   
evaluatorsr   c          
         ddl m} g }|D ]}t          |t          j                  r|                    |           2t          |          r(|                    t          j        |                     it          dt          |                     |pt          j                    }t          | |          } ||          5 } |j        |j        gt          t!          ||           R  }	ddd           n# 1 swxY w Y   |	D ]}
dS )a  Compute test metrics for a given test name using a list of evaluators.

    Args:
        project_name (str): The name of the test project to evaluate.
        evaluators (list): A list of evaluators to compute metrics with.
        max_concurrency (Optional[int], optional): The maximum number of concurrent
            evaluations. Defaults to 10.
        client (Optional[Client], optional): The client to use for evaluations.
            Defaults to None.

    Returns:
        None: This function does not return any value.
    r   )ContextThreadPoolExecutorz5Evaluation not yet implemented for evaluator of type )max_workersN)	langsmithr   
isinstancels_evalRunEvaluatorr/   callablerun_evaluatorNotImplementedErrortyperW   rX   r~   mapevaluate_runzipr   )rP   r   r   r7   r   evaluators_functracesexecutorr1   ro   s              r   compute_test_metricsr      s   * 433333.0K  dG011 	t$$$$d^^ 	w4T::;;;;%TT

TT   -r+--F v66F	"	"	?	?	? 
8(,
"%~fk'J'J"K
 
 

 
 
 
 
 
 
 
 
 
 
 
 
 
 
    s   )C99C= C=)'__doc__rw   rb   r   r)   collections.abcr   typingr   r   langsmith.run_trees	run_treesrW   langsmith.schemasschemas
ls_schemasr   r   r   #langsmith._internal._beta_decoratorr   langsmith.clientr   r,   r   Runr\   r5   r   boolTracerSessionrp   r~   r   r   tupler   intr   r$   r   r   <module>r      s   
           $ $ $ $ $ $ $ $ $ $ $ $ $ $             & & & & & & + + + + + + 9 9 9 9 9 9 # # # # # #4     ,JN  d    8 
 (,#!!h h h
:>
"h h  }	h
 Vh h h h h h hVc 6 d:>>R    $ GCLLGCLL1$q' 1$q' 1d5A;6G 1 1 1 1 
 &(#' ' '' ' c]	'
 V' 
' ' ' ' ' 'r   