U h@sZdZddlmZddlZddlmZddlZddlZddl Z ddl Z ddl m Z m Z mZmZmZmZmZmZmZmZmZmZmZmZddlZddlmZddlmZmZddlmZddlm Z!dd l"m#Z$dd l%m&Z&dd l'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8dd l9m:Z:m;Z;mZ?e?j@Z@ne Z@eAeBZCeeDgeeDfZEdBdddddddddddddddddZFdCddddddddd d! d"d#ZGdDd$dddddddddddd%ddd&d'ZHGd(d)d)e,ZIGd*ddZJd+d,d-d.d/d0d1d2d3ZKd4d+d5d6d7ZLd$d/d8d9d:d;ZMed<ZNd=d>d?d@dAZOdS)EzV2 Evaluation Interface.) annotationsN) TYPE_CHECKINGAny AsyncIterable AsyncIterator AwaitableCallableDictIterableListOptionalSequenceTypeVarUnioncast) run_helpers) run_treesschemas)r)utils)_aiter) _warn_once) AEVALUATOR_TDATA_T EVALUATOR_TExperimentResultRow_ExperimentManagerMixin_extract_feedback_keys_ForwardResults_is_langchain_runnable_load_examples_map_load_experiment _load_tqdm _load_traces _resolve_data_resolve_evaluators_resolve_experiment _to_pandas_wrap_summary_evaluators)SUMMARY_EVALUATOR_TEvaluationResultEvaluationResults RunEvaluatorTzHUnion[DATA_T, AsyncIterable[schemas.Example], Iterable[schemas.Example]]z4Optional[Sequence[Union[EVALUATOR_T, AEVALUATOR_T]]]z'Optional[Sequence[SUMMARY_EVALUATOR_T]]Optional[dict] Optional[str] Optional[int]intOptional[langsmith.Client]boolz6Optional[Union[schemas.TracerSession, str, uuid.UUID]]z%Union[ATARGET_T, AsyncIterable[dict]]AsyncExperimentResults)data evaluatorssummary_evaluatorsmetadataexperiment_prefix descriptionmax_concurrencynum_repetitionsclientblocking experimentupload_resultstargetreturnc  sP| s td| r(|r(td| d|t|||||||||| | | | d IdHS)aEvaluate an async target system or function on a given dataset. Args: target (Union[AsyncCallable[[dict], dict], AsyncIterable[dict]]): The async target system or function to evaluate. data (Union[DATA_T, AsyncIterable[schemas.Example]]): The dataset to evaluate on. Can be a dataset name, a list of examples, an async generator of examples, or an async iterable of examples. evaluators (Optional[Sequence[EVALUATOR_T]]): A list of evaluators to run on each example. Defaults to None. summary_evaluators (Optional[Sequence[SUMMARY_EVALUATOR_T]]): A list of summary evaluators to run on the entire dataset. Defaults to None. metadata (Optional[dict]): Metadata to attach to the experiment. Defaults to None. experiment_prefix (Optional[str]): A prefix to provide for your experiment name. Defaults to None. description (Optional[str]): A description of the experiment. max_concurrency (Optional[int]): The maximum number of concurrent evaluations to run. Defaults to None. num_repetitions (int): The number of times to run the evaluation. Each item in the dataset will be run and evaluated this many times. Defaults to 1. client (Optional[langsmith.Client]): The LangSmith client to use. Defaults to None. blocking (bool): Whether to block until the evaluation is complete. Defaults to True. experiment (Optional[schemas.TracerSession]): An existing experiment to extend. If provided, experiment_prefix is ignored. For advanced usage only. Returns: AsyncIterator[ExperimentResultRow]: An async iterator over the experiment results. Environment: - LANGSMITH_TEST_CACHE: If set, API calls will be cached to disk to save time and cost during testing. Recommended to commit the cache files to your repository for faster CI/CD runs. Requires the 'langsmith[vcr]' package to be installed. Examples: >>> from typing import Sequence >>> from langsmith import Client, aevaluate >>> from langsmith.schemas import Example, Run >>> client = Client() >>> dataset = client.clone_public_dataset( ... "https://smith.langchain.com/public/419dcab2-1d66-4b94-8901-0357ead390df/d" ... ) >>> dataset_name = "Evaluate Examples" Basic usage: >>> def accuracy(run: Run, example: Example): ... # Row-level evaluator for accuracy. ... pred = run.outputs["output"] ... expected = example.outputs["answer"] ... return {"score": expected.lower() == pred.lower()} >>> def precision(runs: Sequence[Run], examples: Sequence[Example]): ... # Experiment-level evaluator for precision. ... # TP / (TP + FP) ... predictions = [run.outputs["output"].lower() for run in runs] ... expected = [example.outputs["answer"].lower() for example in examples] ... # yes and no are the only possible answers ... tp = sum([p == e for p, e in zip(predictions, expected) if p == "yes"]) ... fp = sum([p == "yes" and e == "no" for p, e in zip(predictions, expected)]) ... return {"score": tp / (tp + fp)} >>> import asyncio >>> async def apredict(inputs: dict) -> dict: ... # This can be any async function or just an API call to your app. ... await asyncio.sleep(0.1) ... return {"output": "Yes"} >>> results = asyncio.run( ... aevaluate( ... apredict, ... data=dataset_name, ... evaluators=[accuracy], ... summary_evaluators=[precision], ... experiment_prefix="My Experiment", ... description="Evaluate the accuracy of the model asynchronously.", ... metadata={ ... "my-prompt-version": "abcd-1234", ... }, ... ) ... ) # doctest: +ELLIPSIS View the evaluation results for experiment:... Evaluating over only a subset of the examples using an async generator: >>> async def example_generator(): ... examples = client.list_examples(dataset_name=dataset_name, limit=5) ... for example in examples: ... yield example >>> results = asyncio.run( ... aevaluate( ... apredict, ... data=example_generator(), ... evaluators=[accuracy], ... summary_evaluators=[precision], ... experiment_prefix="My Subset Experiment", ... description="Evaluate a subset of examples asynchronously.", ... ) ... ) # doctest: +ELLIPSIS View the evaluation results for experiment:... Streaming each prediction to more easily + eagerly debug. >>> results = asyncio.run( ... aevaluate( ... apredict, ... data=dataset_name, ... evaluators=[accuracy], ... summary_evaluators=[precision], ... experiment_prefix="My Streaming Experiment", ... description="Streaming predictions for debugging.", ... blocking=False, ... ) ... ) # doctest: +ELLIPSIS View the evaluation results for experiment:... >>> async def aenumerate(iterable): ... async for elem in iterable: ... print(elem) >>> asyncio.run(aenumerate(results)) Running without concurrency: >>> results = asyncio.run( ... aevaluate( ... apredict, ... data=dataset_name, ... evaluators=[accuracy], ... summary_evaluators=[precision], ... experiment_prefix="My Experiment Without Concurrency", ... description="This was run without concurrency.", ... max_concurrency=0, ... ) ... ) # doctest: +ELLIPSIS View the evaluation results for experiment:... Using Async evaluators: >>> async def helpfulness(run: Run, example: Example): ... # Row-level evaluator for helpfulness. ... await asyncio.sleep(5) # Replace with your LLM API call ... return {"score": run.outputs["output"] == "Yes"} >>> results = asyncio.run( ... aevaluate( ... apredict, ... data=dataset_name, ... evaluators=[helpfulness], ... summary_evaluators=[precision], ... experiment_prefix="My Helpful Experiment", ... description="Applying async evaluators example.", ... ) ... ) # doctest: +ELLIPSIS View the evaluation results for experiment:... z&'upload_results' parameter is in beta.zeExpected at most one of 'experiment' or 'experiment_prefix', but both were provided. Got: experiment=z, experiment_prefix=) r4r5r6r7r8r9r:r;r<r=r>r?N)r ValueError _aevaluate) r@r4r5r6r7r8r9r:r;r<r=r>r?rDA/tmp/pip-unpacked-wheel-cqvhoa9t/langsmith/evaluation/_arunner.py aevaluateIs,.rFFz,Union[str, uuid.UUID, schemas.TracerSession]"AsyncIterator[ExperimentResultRow]) r5r6r7r:r< load_nestedr=r>rAc s|p t}t|tjr|ntt||IdH}tjt|||dIdH} tt ||IdHfdd| D} t | | |||||||d IdHS)aL Evaluate existing experiment runs asynchronously. Args: experiment (Union[str, uuid.UUID]): The identifier of the experiment to evaluate. evaluators (Optional[Sequence[EVALUATOR_T]]): Optional sequence of evaluators to use for individual run evaluation. summary_evaluators (Optional[Sequence[SUMMARY_EVALUATOR_T]]): Optional sequence of evaluators to apply over the entire dataset. metadata (Optional[dict]): Optional metadata to include in the evaluation results. max_concurrency (Optional[int]): Optional maximum number of concurrent evaluations. client (Optional[langsmith.Client]): Optional Langsmith client to use for evaluation. load_nested: Whether to load all child runs for the experiment. Default is to only load the top-level root runs. blocking (bool): Whether to block until evaluation is complete. Returns: AsyncIterator[ExperimentResultRow]: An async iterator over the experiment results. Examples: Define your evaluators >>> from typing import Sequence >>> from langsmith.schemas import Example, Run >>> def accuracy(run: Run, example: Example): ... # Row-level evaluator for accuracy. ... pred = run.outputs["output"] ... expected = example.outputs["answer"] ... return {"score": expected.lower() == pred.lower()} >>> def precision(runs: Sequence[Run], examples: Sequence[Example]): ... # Experiment-level evaluator for precision. ... # TP / (TP + FP) ... predictions = [run.outputs["output"].lower() for run in runs] ... expected = [example.outputs["answer"].lower() for example in examples] ... # yes and no are the only possible answers ... tp = sum([p == e for p, e in zip(predictions, expected) if p == "yes"]) ... fp = sum([p == "yes" and e == "no" for p, e in zip(predictions, expected)]) ... return {"score": tp / (tp + fp)} Load the experiment and run the evaluation. >>> from langsmith import aevaluate, aevaluate_existing >>> dataset_name = "Evaluate Examples" >>> async def apredict(inputs: dict) -> dict: ... # This can be any async function or just an API call to your app. ... await asyncio.sleep(0.1) ... return {"output": "Yes"} >>> # First run inference on the dataset ... results = asyncio.run( ... aevaluate( ... apredict, ... data=dataset_name, ... ) ... ) # doctest: +ELLIPSIS View the evaluation results for experiment:... Then evaluate the results >>> experiment_name = "My Experiment:64e6e91" # Or manually specify >>> results = asyncio.run( ... aevaluate_existing( ... experiment_name, ... evaluators=[accuracy], ... summary_evaluators=[precision], ... ) ... ) # doctest: +ELLIPSIS View the evaluation results for experiment:... N)rHcsg|]}|jqSrD)reference_example_id).0runZdata_maprDrE gsz&aevaluate_existing..)r4r5r6r7r:r<r=r>) rget_cached_client isinstancerZ TracerSession aitertools aio_to_threadr r"rrC) r>r5r6r7r:r<rHr=projectrunsr4rDrLrEaevaluate_existings0N   rT-Union[DATA_T, AsyncIterable[schemas.Example]]zdntt t j |}t t| || IdH\}}t|| ||pv||||| dIdH}td}|dk r|IdH}t||d}nd}tj|| jgd|| r|jtt||dIdH}|r|j||dIdH}|r*||IdH}t|}| rF|IdH|W5QRSQRXdS)N __aiter__)r<r7r>r9r;rSr?z.yaml)Z ignore_hostsr:)asyncioiscoroutinefunctionhasattr iscoroutinerVrrtrNrr rRunrPrQr%_AsyncExperimentManagerastartls_utilsZ get_cache_dirget_dataset_idpathlibPathZwith_optional_cacheZapi_urlawith_predictions ATARGET_Tawith_evaluatorsawith_summary_evaluatorsr3wait)r@r4r5r6r7r8r9r:r;r<r=r>r?Zis_async_targetrSZ experiment_manager cache_dirZdsid cache_pathresultsrDrDrErCusZ     rCc sNeZdZdZdIddddd d d d d d d fdd ZddddZddddZddddZddddZddddZ dJd d!dd"d#d$Z dd%d&d dd'd(d)Z d*dd+d,d-Z d.dd/d0Z d1dd2d3ZdKd d!d4d"d5d6ZdLd7d d.d'd8d9Zd7d:d;d:d<d=d>Zd*dd+d?d@Zd ddAdBZdCddDdEZdFddGdHZZS)Mr^aaManage the execution of experiments asynchronously. Supports lazily running predictions and evaluations in parallel to facilitate result streaming and early debugging. Args: data (DATA_T): The data used for the experiment. Can be a dataset name or ID OR a generator of examples. runs (Optional[Iterable[schemas.Run]]): The runs associated with the experiment predictions. experiment (Optional[schemas.TracerSession]): The tracer session associated with the experiment. experiment_prefix (Optional[str]): The prefix for the experiment name. description (Optional[str]): The description for the experiment. metadata (Optional[dict]): Additional metadata for the experiment. client (Optional[langsmith.Client]): The Langsmith client used for the experiment. evaluation_results (Optional[Iterable[EvaluationResults]]): The evaluation sresults for the experiment. summary_results (Optional[Iterable[EvaluationResults]]): The aggregate results for the experiment. Nr,Tz+Optional[Union[schemas.TracerSession, str]]r-zBOptional[Union[Iterable[schemas.Run], AsyncIterable[schemas.Run]]]r1z*Optional[AsyncIterable[EvaluationResults]]r.r0r2rU) r>r7rSr<evaluation_resultssummary_resultsr9r;r?r4c  sTtj||||d||_d|_|dk r2t|nd|_||_||_| |_ | |_ dS)N)r>r7r<r9) super__init___data _examplesrPensure_async_iterator_runs_evaluation_results_summary_results_num_repetitions_upload_results) selfr4r>r7rSr<rmrnr9r;r? __class__rDrErpsz _AsyncExperimentManager.__init__AsyncIterator[schemas.Example]rAcsb|jdkrr7r<rSrmr?)rPrrStopAsyncIterationrBrxZ _get_projectZ_print_experiment_startrw _metadatar{r<rtru)ryZ first_examplerRrDrDrEr_s,    z_AsyncExperimentManager.astartr/re)r:r@rAcsV|j||d}tj|dtd\}}tdd|2|j|j|jdd|2|j dS)NrWrrcSs |z3dHW}|dVq6dSNrrDrJpredrDrDrE 9s z<_AsyncExperimentManager.awith_predictions..cSs |z3dHW}|dVq6dSNrKrDrrDrDrEr=s )r>r7r<rSr?) _apredictrPrrXrr^rrr<rx)ryr@r:Z_experiment_resultsr1r2rDrDrErd0s  z)_AsyncExperimentManager.awith_predictionsrWz*Sequence[Union[EVALUATOR_T, AEVALUATOR_T]])r5r:rAc spt|}|j||d}tj|dtd\}}}tdd|2|j|j|j dd|2dd|2|j |j dS) NrWrcSs |z3dHW}|dVq6dSrrDrJrrDrDrErKs z;_AsyncExperimentManager.awith_evaluators..cSs |z3dHW}|dVq6dSrrDrrDrDrErOs cSs |z3dHW}|dVq6dS)NrmrDrrDrDrErPs r>r7r<rSrmrnr?) r$_ascorerPrrXrr^rrr<rvrx)ryr5r:Zexperiment_resultsrrZr3rDrDrErfAs   z(_AsyncExperimentManager.awith_evaluatorszSequence[SUMMARY_EVALUATOR_T])r6rAc sBt|}||}t|IdH|j|j|j||j||j dS)Nr) r'_aapply_summary_evaluatorsr^rrrr<rrurx)ryr6Zwrapped_evaluatorsZaggregate_feedback_genrDrDrErgUs  z0_AsyncExperimentManager.awith_summary_evaluatorsrGcCsJt||IdH|2z"3dHW\}}}t|||dVq 6dS)NrKrrm)rP async_ziprrrr)ryrKrrmrDrDrE aget_resultsfs z$_AsyncExperimentManager.aget_resultszDict[str, List[dict]]cs,|jdkrdgiSddd|j2IdHiS)Nrlcs(g|z3dHW}|dD]}|qq6S)NrlrD)rJrlresrDrDrErMts z?_AsyncExperimentManager.aget_summary_scores..)rvryrDrDrEaget_summary_scoresps  z+_AsyncExperimentManager.aget_summary_scoreszAsyncIterator[_ForwardResults]csPt|fdd}tj||dd2z3dHW}|Vq(6IdHdS)Ncs:IdH2z$3dHW}t|jjjVq6dSN)r _aforwardexperiment_namerr<)rfnryrDrE predict_allsz6_AsyncExperimentManager._apredict..predict_allMbP?Z_eager_consumption_timeout)_ensure_async_traceablerPaiter_with_concurrency_aend)ryr@r:rrrDrrEr}s z!_AsyncExperimentManager._apredictzSequence[RunEvaluator]c sTtjdd>fdd}tj||dd2z3dHW}|Vq06W5QRXdS)N) max_workerscs.2z3dHW}j|dVq6dS)N)executor)r_arun_evaluators)current_resultsr5rryrDrE score_alls z2_AsyncExperimentManager._ascore..score_allrr)cfZThreadPoolExecutorrPr)ryr5r:rrrDrrErsz_AsyncExperimentManager._ascorerzcf.ThreadPoolExecutor)r5rrrAc st}|dpid|ji}tjf|d||js4dnd|jd|d|d}|d }|D]P}zF|j|d IdH} |d |j| |jr|jj | |d Wqdt k rzzVt |} t fd d| Dd} |d |j| |jr|jj | |d Wn4t k rN} zt d| W5d} ~ XYnXt jdt|djdtddt jdt|djdtddW5dXYqdXqdt||dW5QRSQRXdS)Nr7r>r5localT project_namer7enabledr<rKrrmrKrrl)rKZ _executorcs&g|]}t|jtddidqS)errorT)keyZ source_run_idcommentextra)r)idrepr)rJrerKrDrErMsz<_AsyncExperimentManager._arun_evaluators..)rlzError parsing feedback keys: zError running evaluator z on run : exc_infor)rhget_tracing_contextrtracing_contextrxr<Z aevaluate_runextend_select_eval_resultsZ_log_evaluation_feedback Exceptionrr*loggerdebugrrrr) ryr5rrcurrent_contextr7rZ eval_results evaluatorZevaluator_responseZ feedback_keysZerror_responsee2rDrrErs~           z(_AsyncExperimentManager._arun_evaluatorsc Csgg}}t|IdH}t||2z$3dHW\}}||||q.6g}|jrj|jnd}t } | dpi|j |d} t j f| d| |jsdnd|j d|D]} zz| ||} |j j| | jd} || |jr6| D]B}|jdhd }|d d}tj|j jf|d||d IdHqWqtk rz}z"tjd t| d |ddW5d}~XYqXqW5QRXd|iVdS)Nr7)r>Z experiment_idr5rTr)fn_nameZ target_run_id)excludeevaluator_info)Zrun_id project_idZ source_infoz Error running summary evaluator rrrl)rPrsrrrappendrxZ_get_experimentrrrrrr<r__name__rdictpoprQZcreate_feedbackrrrr)ryr6rSZexamplesZasync_examplesrKrZaggregate_feedbackrrr7rZsummary_eval_resultZflattened_resultsrZfeedbackrrrDrDrErsd        $z2_AsyncExperimentManager._aapply_summary_evaluatorscsTg}|IdH2z3dHW}|jr||jq6|r@t|nd}|rP|SdSr)r modified_atrmax isoformat)ryrrZmax_modified_atrDrDrE_get_dataset_version's z,_AsyncExperimentManager._get_dataset_versionzOptional[list[str]]cs~t}|IdH2z^3dHW}|jrh|jdrht|jdtrh|jdD]}t|trL||qLq|dq6t|S)NZ dataset_splitbase)setrr7getrOlistradd)rysplitsrsplitrDrDrE_get_dataset_splits2s  z+_AsyncExperimentManager._get_dataset_splitsNonecs||js dS|j}|dkr td|}|IdH|d<|IdH|d<|jj|j|j pjt j t j j |j|ddS)NzExperiment not started yet.Zdataset_versionZdataset_splits)end_timer7)rxrrBZ_get_experiment_metadatarrr<Zupdate_projectrrdatetimenowtimezoneutcr7)ryr>Zproject_metadatarDrDrErBs z_AsyncExperimentManager._aend) NNNNNNNr,T)N)N)N)r __module__ __qualname____doc__rprrarrr_rdrfrgrrrrrrrrr __classcell__rDrDrzrEr^s@&     M7 r^c@seZdZddddZeddddZd dd d Zd dd dZdddddZd#ddddddZ ddddZ ddddZ dddd Z ddd!d"Z dS)$r3r^)experiment_managercCs4||_g|_t|_t||j|_d|_dS)Nr) _manager_resultsrXr_lockZ create_task _process_data_task_processed_count)ryrrDrDrErpXs  zAsyncExperimentResults.__init__rr}cCs|jjSr)rrrrDrDrErbsz&AsyncExperimentResults.experiment_namerGcCs|SrrDrrDrDrErVfsz AsyncExperimentResults.__aiter__rc sdddfdd }j4IdHTjtjkrdjj}jd7_|W5QIdHRSjrrtW5QIdHRXttj |tjddIdHqdS)Nr0r)indexrAcs j|krtdIdHqdS)Ng?)rrXsleep)rrrDrE_wait_until_indexjs z;AsyncExperimentResults.__anext__.._wait_until_indexr,)timeout) rrlenrrdonerrXZshieldwait_for)ryrrrDrrE __anext__is  z AsyncExperimentResults.__anext__r)rirAc st}||2z83dHW}|j4IdH|j|W5QIdHRXq6|IdH}|j4IdH ||_W5QIdHRXdSr)r!rrrrrrv)ryriZtqdmitemZsummary_scoresrDrDrEr{s z$AsyncExperimentResults._process_datarNr/ DataFrame)startendrAcCst|j||dS)N)rr)r&r)ryrrrDrDrE to_pandassz AsyncExperimentResults.to_pandascCs:ddl}|jr.|jdr.|dd}|S|SdS)Nrpandas)importlib.utilrutil find_specr _repr_html___repr__)ry importlibZdfrDrDrEr s  z"AsyncExperimentResults._repr_html_r0cCs t|jSr)rrrrDrDrE__len__szAsyncExperimentResults.__len__cCsd|jdS)Nz)rrrDrDrEr szAsyncExperimentResults.__repr__cs|jIdHdSr)rrrDrDrErhszAsyncExperimentResults.wait)rN)rrrrppropertyrrVrrrr rr rhrDrDrDrEr3Ws   z,rh.SupportsLangsmithExtra[[dict], Awaitable]zschemas.Examplerrzlangsmith.Clientr)rrrr7r<rAc sddddfdd }tjddzH||jtj|j|||d|jrN|jn|ji|d d IdHWn8tk r}zt j d |dd d W5d}~XYnXt t t j|dW5QRSQRXdS)Nzrun_trees.RunTreer)rrAcs|dSrrD)rrKrDrE_get_runsz_aforward.._get_runT)rZexample_version)rIZon_endrr7r<)Zlangsmith_extrazError running target function: r,)r stacklevelr)rrinputsZLangSmithExtrarrrZ created_atrrrrrrr])rrrr7r<rrrDrrErs:  rre)r@rAcCs\t|s,t|s,t|r$tdntdt|r:|St|rH|j}tjdd|SdS)NzTarget must be an async function. For sync functions, use evaluate. Example usage: async def predict(inputs: dict) -> dict: # do work, like chain.invoke(inputs) return {...} await aevaluate(predict, ...)zTarget must be a callable async function. Received a non-callable object. Example usage: async def predict(inputs: dict) -> dict: # do work, like chain.invoke(inputs) return {...} await aevaluate(predict, ...)Z AsyncTarget)name) rXrYrcallablerBrZis_traceable_functionZainvokeZ traceable)r@rDrDrErs  rr|)r4r<rAcCs&t|trt|Stt||dS)z*Return the examples for the given dataset.r~)rOrrPrsr#)r4r<rDrDrErs  rTzIterable[AsyncIterable[T]]zAsyncIterator[T])iterablerAcCs(|D]}|2z3dHW}|Vq 6qdS)zChain multiple async iterables.NrD)rZ sub_iterablerrDrDrErsr) NNNNNNr,NTNT)NNNNNFT) NNNNNNr,NTNT)Pr __future__rrXconcurrent.futuresZfuturesrrloggingrbuuidtypingrrrrrrr r r r r rrrZ langsmithrrrrr\rr`Zlangsmith._internalrrPZ#langsmith._internal._beta_decoratorrZlangsmith.evaluation._runnerrrrrrrrrrr r!r"r#r$r%r&r'Zlangsmith.evaluation.evaluatorr(r)r*r+rpdr getLoggerrrrrerFrTrCr^r3rrrrrrDrDrDrEs  @     L *J j*>'E*