U h@sFdZddlmZddlZddlZddlZddlZddlZddl Z ddl m Z m Z ddl m Z mZmZmZmZmZmZmZmZddlmZddlmZddlmZdd lmZmZdd lm Z m!Z!dd l"m#Z#m$Z$m%Z%dd l"m&Z'dd l"m(Z)ddl*m+Z+m,Z,ddl-m.Z.ddl/m0Z0ddl1m2Z2m3Z3ddl4m5Z5m6Z6ddl4m7Z8ddl9m:Z:m;Z;ddlZ>m?Z?m@Z@mAZAddlBmCZCddlDmEZEddlFmGZGddlHmIZIddlJmKZKddlLmMZMmNZNmOZOddlPmQZRdd lSm&ZTddlSmUZUmVZVe rddlWZXeYeZZ[eegeeIe#ffeee\gefe#eIfZ]eegeeIe#ffefZ^Gddde_Z`Gd d!d!e\ZaGd"d#d#e\Zbdd%d&d'd(d)d*Zcd+d&d,d-d.ZdGd/d0d0eGZed+d1d,d2d3Zfd4d5d6d7d8d9Zgd4d:d5d6d;dd?d@Zid'dAdBdCdDdEdFdGZjdHdIdJdKdLdMZkdHdIdJdNdOdPZldHdIdJdQdRdSZmdTdUd&dCdIdJdJdJdVdW dXdYZndHdIdIdIdZd[d\d]ZodHd&dCdIdIdId^d_d`daZpdddddbdcd+dIddd5dedfdgdhdiZqddddjdkd+dddId5dedldmdndoZrddpd4dqd'd5drdsdtduZsddddjdcd+dddId5dedfdvdwdxZtddddjdkd+dddId5dedydmdzd{Zuddpd4dqd'd5drdsd|d}Zvdd~d&d%d&dedIdddddZwGdddeGddZxejyGdddZzddddZ{dd6dddZ|dZ}dddddddddd&d%dBdddJdeddJdd+d ddZ~dddddddddd&d%dBdddJdeddJdd+d ddZdZee_edde~_dS)z>Utilities for running language models or Chains over datasets.) annotationsN)datetimetimezone) TYPE_CHECKINGAnyCallableDictListOptionalTupleUnioncast)warn_deprecated) Callbacks)BaseLanguageModel) BaseMessagemessages_from_dict) ChatResult LLMResult)RunnableRunnableConfigRunnableLambdaconfig)utils)EvaluatorCallbackHandlerwait_for_all_evaluators)LangChainTracer)Client) get_git_infoget_langchain_env_var_metadata)EvaluationResult RunEvaluator) run_evaluator) as_runnableis_traceable_function)DatasetDataTypeExampleRun TracerSession)LangSmithError) HTTPError) TypedDict)Chain)load_evaluator) EvaluatorTypePairwiseStringEvaluatorStringEvaluator) evaluation)name_generationprogressc@seZdZdZdS)InputFormatErrorz(Raised when the input format is invalid.N)__name__ __module__ __qualname____doc__r;r;K/tmp/pip-unpacked-wheel-bo69hh5q/langchain/smith/evaluation/runner_utils.pyr6Nsr6c@s,eZdZdZddddZddddZdS) TestResultz1A dictionary of the results of a single test run. pd.DataFramereturncCs.|}dd|jD}|jddj|ddS)zReturn quantiles for the feedback scores. This method calculates and prints the quantiles for the feedback scores across all feedback keys. Returns: A DataFrame containing the quantiles for each feedback key. cSs6g|].}|ds.|ds.|dks.|dr|qS)inputs.outputs.>outputinput reference) startswith).0colr;r;r< es    z5TestResult.get_aggregate_feedback..all)include)Zaxis) to_dataframecolumnsZdescribeZdrop)selfZdfZto_dropr;r;r<get_aggregate_feedbackXs z!TestResult.get_aggregate_feedbackc CsJz ddl}Wn,tk r8}ztd|W5d}~XYnXg}g}|dD]\}}|d}|d}t|trdd|D} n|dkri} nd|i} d d|d D| } d |krt|d tr| d d|d Dn |d | d <| d d|D|d|d|dd|| ||qN|j||dS)z#Convert the results to a dataframe.rNzfPandas is required to convert the results to a dataframe. to install pandas, run `pip install pandas`.resultsfeedbackrCcSsi|]\}}d||qS)rBr;rGkvr;r;r< sz+TestResult.to_dataframe..cSsi|]\}}d||qS)rAr;rSr;r;r<rVsrDrEcSsi|]\}}d||qS)z reference.r;rSr;r;r<rVscSsi|]}d|j|jqS)z feedback.)keyZscore)rGfr;r;r<rVs Errorexecution_timerun_id)errorrZr[)index) pandas ImportErroritemsget isinstancedictupdateappendZ DataFrame) rOpdeindicesrecords example_idresultrRZoutput_rCrr;r;r<rMosJ       zTestResult.to_dataframeN)r7r8r9r:rPrMr;r;r;r<r=Usr=cs:eZdZdZddddfdd Zddd d d ZZS) EvalErrorz"Your architecture raised an error. BaseExceptionrNone)rYkwargsr@c stjfd|i|dS)NrY)super__init__)rOrYrp __class__r;r<rrszEvalError.__init__str)namer@cCs4z ||WStk r.td|dYnXdS)Nz%'EvalError' object has no attribute '')KeyErrorAttributeError)rOrvr;r;r< __getattr__s zEvalError.__getattr__)r7r8r9r:rrrz __classcell__r;r;rsr<rmsrm MODEL_OR_CHAIN_FACTORYruMCF)llm_or_chain_factory dataset_namer@csft|trR|jj}|jdk rFjjj}td|d|d|dfddSt|tr`|St|trz|fddSt|rbt |rt t t |fd dSz |}WnPt k rt t |}t|}td |d t|fd dYSXt t |t|tr|St t t |rFt t t |fd dSt|ts^fddSS|S)zForgive the user if they pass in a chain without memory instead of a chain factory. It's a common mistake. Raise a more helpful error message as well.Na$Cannot directly evaluate a chain with stateful memory. To evaluate this chain, pass in a chain constructor that initializes fresh memory each time it is called. This will safegaurd against information leakage between dataset examples. For example: def chain_constructor(): new_memory = z(...) return z*(memory=new_memory, ...) run_on_dataset("z", chain_constructor, ...)csSNr;r;)chainr;r<z(_wrap_in_chain_factory..csSrr;r;)lcfr;r<rrcsSrr;r; runnable_r;r<rrzWrapping function z as RunnableLambda.csSrr;r;)wrappedr;r<rrcsSrr;r;rr;r<rrcstSr)rr;) constructorr;r<rr)rbr.rtr7Zmemory ValueErrorrrcallabler%r$r r TypeErrorinspect signatureloggerinfor)rrZ chain_classZ memory_class_modelZ user_funcsigr;)rrrrrr<_wrap_in_chain_factorysH                 rzDict[str, Any])inputsr@cCs4|s tdg}d|krJt|dts>tdt|dj|dg}nd|krt|dtrvtdd|dDstdt|dj|d}nnt|dkrtt | }t|tr|g}n0t|trtd d|Dr|}ntd |ntd |t|dkr|d Std t|ddS)zGet prompt from inputs. Args: inputs: The input dictionary. Returns: A string prompt. Raises: InputFormatError: If the input format is invalid. Inputs should not be empty.promptz"Expected string for 'prompt', got promptscss|]}t|tVqdSrrbrurGir;r;r< sz_get_prompt..z,Expected list of strings for 'prompts', got rLcss|]}t|tVqdSrrrr;r;r<r sz)LLM Run expects string prompt input. Got z5LLM Run expects 'prompt' or 'prompts' in inputs. Got rz)LLM Run expects single prompt input. Got z prompts.N) r6rbrutyper7listrJlennextitervalues)rrZprompt_r;r;r< _get_prompts@     rc@seZdZUdZded<dS)ChatModelInputzVInput for a chat model. Parameters: messages: List of chat messages. zList[BaseMessage]messagesNr7r8r9r:__annotations__r;r;r;r<rs rrccCs|s td|}d|kr,|d|d<n t|dkrLtt||d<d|kr|d}t|tr~t dd|Dr~|g}t|dkrt |d|d<ntd|Std |d S) zGet Chat Messages from inputs. Args: inputs: The input dictionary. Returns: A list of chat messages. Raises: InputFormatError: If the input format is invalid. rrrDrLcss|]}t|tVqdSr)rbrcrr;r;r<r9sz _get_messages..rzGBatch messages not supported. Please provide a single list of messages.zMChat Run expects single List[dict] or List[List[dict]] 'messages' input. Got N) r6copypoprrrrrbrrJr)rZ input_copyZ raw_messagesr;r;r< _get_messages%s,   rr(Optional[Callable[[Dict], Any]]ro) first_example input_mapperr@c Cs|rP||j}t|tst|tr4tdd|Dstd|dt|dnZzt|jWnJtk rzt|jWn&tk rtd|jdYnXYnXdS)Ncss|]}t|tVqdSrrbrrGmsgr;r;r<rUsz>_validate_example_inputs_for_language_model..zWhen using an input_mapper to prepare dataset example inputs for an LLM or chat model, the output must a single string or a list of chat messages. Got: of type .zvExample inputs do not match language model input format. Expected a dictionary with messages or a single prompt. Got: z Please update your dataset OR provide an input_mapper to convert the example.inputs to a compatible format for the llm or chat model you wish to evaluate.) rrbrurrJr6rrr)rrZ prompt_inputr;r;r<+_validate_example_inputs_for_language_modelMs&   rr.)rrrr@cCs|rb||j}t|j|}t|tsBtd|dt|d|rtd|jd|nP|j}t|j|}t |dkrt |jdkrn|rtd|jd|dS) z.)rrrzUnsupported custom evaluator: z+. Expected RunEvaluator or StringEvaluator.) evaluatorscustom_evaluatorsanyrrrrerbr"r2rrrrrr)rrrrrrrrrrrr#rZcustom_evaluatorr;r;r<rUs\          rr callbacksrmetadatarrOptional[Dict[str, Any]]zUnion[str, BaseMessage])rrrrrrr@c s|dk rn||}t|ts6t|tr\tdd|Dr\|j|t||pFg|pLiddIdHStd|dn|z2t|}|j|t||pg|piddIdH}WnHtk rt|} |jf| dt||pg|pidiIdH}YnX|S) aAsynchronously run the language model. Args: llm: The language model to run. inputs: The input dictionary. tags: Optional tags to add to the run. callbacks: Optional callbacks to use during the run. input_mapper: Optional function to map inputs to the expected format. Returns: The LLMResult or ChatResult. Raises: ValueError: If the LLM type is unsupported. InputFormatError: If the input format is invalid. Ncss|]}t|tVqdSrrrr;r;r<rsz_arun_llm..rrrrz%Input mapper returned invalid format 3 Expected a single string or list of chat messages.r) rbrurrJainvokerr6rr) rrrrrrprompt_or_messagesr llm_output llm_inputsr;r;r< _arun_llmsR  rrrrzUnion[Chain, Runnable]zUnion[dict, str])rrrrrrr@c s|dkr |n||}t|trrt|trrt|dkrr|jrrtt|}|j|t ||pZg|p`iddIdH}n*t |pzg||pid} |j|| dIdH}|S)z%Run a chain asynchronously on inputs.NrLrrrrr) rbr.rcrrrrrrr rrrrrrZinputs_valrCrunnable_configr;r;r< _arun_chains2  r)rrz'Union[dict, str, LLMResult, ChatResult])rrrrr@c st|trdnd}d}znt|trNt||j|d|d||ddIdH}n0|}t||j|d|d||ddIdH}|}WnTtk r}z6t|d|j d |jd t |t |d }W5d}~XYnX|S) aAsynchronously run the Chain or language model. Args: example: The example to run. llm_or_chain_factory: The Chain or language model constructor to run. tags: Optional tags to add to the run. callbacks: Optional callbacks to use during the run. input_mapper: Optional function to map the input to the expected format. Returns: A list of outputs. LLMr.Nrrrr failed for example with inputs  rY) rbrrrrar Exceptionrridreprrm) rrrr chain_or_llmrkrCrrgr;r;r<_arun_llm_or_chains:    r)rrrrrrr@c Cs|dk rj||}t|ts6t|trXtdd|DrX|j|t||pFg|pLidd}qtd|dnjz,t|}|j|t||pg|pidd}Wn<tk rt|} |jf| dt||pid i}YnX|S) a Run the language model on the example. Args: llm: The language model to run. inputs: The input dictionary. callbacks: The callbacks to use during the run. tags: Optional tags to add to the run. input_mapper: function to map to the inputs dictionary from an Example Returns: The LLMResult or ChatResult. Raises: ValueError: If the LLM type is unsupported. InputFormatError: If the input format is invalid. Ncss|]}t|tVqdSrrrr;r;r<rOsz_run_llm..rrz'Input mapper returned invalid format: rr)rr) rbrurrJinvokerr6rr) rrrrrrrrZ llm_promptsrr;r;r<_run_llm1sJ  rzUnion[Dict, str]c Cs|dkr |n||}t|trlt|trlt|dkrl|jrltt|}|j|t ||pZg|p`idd}n$t |ptg||p|id} |j|| d}|S)zRun a chain on inputs.NrLrrr) rbr.rcrrrrrrrrr;r;r< _run_chainos2  rc Cst|trdnd}d}zbt|trHt||j|d|d||dd}n*|}t||j|d|d||dd}|}Wn`tk r}zBt|j} t |d|j d |jd | d | t |d }W5d}~XYnX|S) a Run the Chain or language model synchronously. Args: example: The example to run. llm_or_chain_factory: The Chain or language model constructor to run. tags: Optional tags to add to the run. callbacks: Optional callbacks to use during the run. Returns: Union[List[dict], List[str], List[LLMResult], List[ChatResult]]: The outputs of the model or chain. rr.Nrrrrrrz Error Type: z , Message: r) rbrrrrarrrr7rrrrm) rrrrrrkrCrrgZ error_typer;r;r<_run_llm_or_chains<   "rrzOptional[Union[str, datetime]]z1Tuple[MCF, TracerSession, Dataset, List[Example]])clientrr project_nameproject_metadatardataset_versionr@c Csjt||}|j|d}t|j|j|d} | s>td|ddd| D} | rXt| nd} | rh| nd} zJ|pti}t} | r|d| i}| |d<|j ||j|rd |ini|d }Wnnt tt fk r$}zHd t |kr|t }d |d |d|d}td|d|W5d}~XYnX|jd|j}td|d|d|d|jdd|||| fS)N)r)Z dataset_idZas_ofzDataset z has no example rows.cSsg|]}|jr|jqSr;) modified_at)rGexr;r;r<rIsz%_prepare_eval_run..gitrr)Zreference_dataset_idZ project_extrarzalready exists z+ run_on_dataset( ... project_name="z - z", # Update since z already exists ) z Test project z/ already exists. Please use a different name: z/compare?selectedSessions=z)View the evaluation results for project 'z' at: z View all tests for Dataset z at: T)flush)rZ read_datasetrZ list_examplesrrmax isoformatrZcreate_projectr,r+ruuuidZuuid4urlprint)rrrrrrr wrapped_modeldatasetrrZmax_modified_atZinferred_versionZgit_infoprojectrguidZ example_msgZcomparison_urlr;r;r<_prepare_eval_runsV    rc@s*eZdZUdZded<ded<ded<dS) _RowResultz5A dictionary of the results for a single example row.z Optional[List[EvaluationResult]]rRzOptional[float]rZrr[Nrr;r;r;r<rs rF)totalc@seZdZUdZded<ded<ded<ded <d ed <d Zd ed<ddddddZdddddZddddZddd d!d"Z d5dd$dd%d&d'Z e d6dd)d*d+d,d-d.d/d0d+d1dd2 d3d4Z d S)7_DatasetRunContainerz3A container to help manage the state of a eval run.rrr*rr~r rrzList[RunnableConfig]configsNz6Optional[List[smith_eval_config.BATCH_EVALUATOR_LIKE]]batch_evaluatorsrzDict[str, _RowResult]rc) batch_resultsall_eval_resultsr@cCsi}t|j|D]\}}tt|t|ji}|j|dg|d|dd|t|j<t|t r~|j |t|jd<n||t|jd<|j r|j |t|jd<q|S)NrRrZr[)rDrRrZr[rYrCrE) ziprr rrarurrrbrmrYr)rOrrrQrrCZ row_resultr;r;r<_merge_test_outputss  z(_DatasetRunContainer._merge_test_outputszDict[str, Run]z List[dict])runsr@c s|j}|sgSfdd|jD}g}tj}|D]}zR|||j}t|trZ|}|t t||j |j j f|d|j jdWq6tk r}ztdt|d|W5d}~XYq6Xq6W5QRX|S)Ncsg|]}t|jqSr;)rurrGrrr;r<rI1sz>_DatasetRunContainer._run_batch_evaluators..)r[Z project_idzError running batch evaluator z: )rr concurrentZfuturesZThreadPoolExecutorrbr!rcrer ZsubmitrZcreate_feedbackrrrrr\r) rOrrZ runs_listaggregate_feedbackexecutorZ evaluatorrkrgr;rr<_run_batch_evaluators-s0   "z*_DatasetRunContainer._run_batch_evaluatorsz,Tuple[Dict[str, _RowResult], Dict[str, Run]]r?c Csi}i}|jD]}tt|dD]}t|trf|j}|D]&\\}}}|t|i d|iq.)rrr rrr)r4Z random_namerdrrrar`rerrrr'kvrr5ZProgressBarCallbackrr)clsrrrrr3rrr6rr7rr rrrTrUrr;r:r<preparesP   z_DatasetRunContainer.prepare)F)NNNr3NNN) r7r8r9r:rrrr!r+r,r2 classmethodr>r;r;r;r<r s*  rr-r?cCsJz.ddlm}|}|dk o,dtt|kWStk rDYdSXdS)Nr) get_ipythonZzmqshellF)ZIPythonr@rurr_)r@resr;r;r<_is_jupyter_environments  rBr>)aggregate_resultsr@cCsRtr,ddlm}m}||d||n"|jdddd}tdt|dS) Nr)HTMLdisplayz

Experiment Results:

cSs|dS)Nz.2fr;)xr;r;r<rrz,_display_aggregate_results..right)Z float_formatZjustifyz Experiment Results:)rBZIPython.displayrDrEZ to_stringr )rCrDrEZformatted_stringr;r;r<r/s  r/aThe input_mapper argument is deprecated and will be removed in a future release. Please add a RunnableLambda to your chain to map inputs to the expected format instead. Example: def construct_chain(): my_chain = ... input_mapper = {'other_key': 'MyOtherInput', 'my_input_key': x} return input_mapper | my_chain run_on_dataset(..., llm_or_chain_factory=construct_chain) (See https://api.python.langchain.com/en/latest/schema/langchain.schema.runnable.base.RunnableLambda.html)r3)r3rr6rrr.r7zOptional[Client]r5r4r) rrrr3rr6rrr.r7rpr@c s| dd} | rtdtdd| dkr2td} | dd} | rPtdddd| rntdd | d dd |pvt}tj|||||| | ||| |d } t j | j d dft t jt| j| d| j| j IdH}| j||dS)Nr0.0.305Tmessagependingr7r0.1.9qThe tags argument is deprecated and will be removed in a future release. Please specify project_metadata instead.PThe following arguments are deprecated and will be removed in a future release: rrJZremovalrr7rrr9rrr.)rr_INPUT_MAPPER_DEP_WARNINGr rarrrr>runnable_utilsZgather_with_concurrencyrmap functoolspartialrr rr2)rrrr3rr6rrr.r7rprr containerrr;r;r<arun_on_datasets\      rYc s| ddrtdtdd| dd} | r.rQrR)rrrSr rarrrr>rrrrZget_executor_for_configrrUrVrWrr r2)rrrr3rr6rrr.r7rprrr r;rZr<run_on_dataset9sb         r[a1 Run the Chain or language model on a dataset and store traces to the specified project name. Args: dataset_name: Name of the dataset to run the chain on. llm_or_chain_factory: Language model or Chain constructor to run over the dataset. The Chain constructor is used to permit independent calls on each example without carrying over state. evaluation: Configuration for evaluators to run on the results of the chain concurrency_level: The number of async tasks to run concurrently. project_name: Name of the project to store the traces in. Defaults to {dataset_name}-{chain class name}-{datetime}. project_metadata: Optional metadata to add to the project. Useful for storing information the test variant. (prompt version, model version, etc.) client: LangSmith client to use to access the dataset and to log feedback and run traces. verbose: Whether to print progress. tags: Tags to add to each run in the project. revision_id: Optional revision identifier to assign this test run to track the performance of different versions of your system. Returns: A dictionary containing the run's project name and the resulting model outputs. For the (usually faster) async version of this function, see :func:`arun_on_dataset`. Examples -------- .. code-block:: python from langsmith import Client from langchain_openai import ChatOpenAI from langchain.chains import LLMChain from langchain.smith import smith_eval.RunEvalConfig, run_on_dataset # Chains may have memory. Passing in a constructor function lets the # evaluation framework avoid cross-contamination between runs. def construct_chain(): llm = ChatOpenAI(temperature=0) chain = LLMChain.from_string( llm, "What's the answer to {your_input_key}" ) return chain # Load off-the-shelf evaluators via config or the EvaluatorType (string or enum) evaluation_config = smith_eval.RunEvalConfig( evaluators=[ "qa", # "Correctness" against a reference answer "embedding_distance", smith_eval.RunEvalConfig.Criteria("helpfulness"), smith_eval.RunEvalConfig.Criteria({ "fifth-grader-score": "Do you have to be smarter than a fifth grader to answer this question?" }), ] ) client = Client() run_on_dataset( client, dataset_name="", llm_or_chain_factory=construct_chain, evaluation=evaluation_config, ) You can also create custom evaluators by subclassing the :class:`StringEvaluator ` or LangSmith's `RunEvaluator` classes. .. code-block:: python from typing import Optional from langchain.evaluation import StringEvaluator class MyStringEvaluator(StringEvaluator): @property def requires_input(self) -> bool: return False @property def requires_reference(self) -> bool: return True @property def evaluation_name(self) -> str: return "exact_match" def _evaluate_strings(self, prediction, reference=None, input=None, **kwargs) -> dict: return {"score": prediction == reference} evaluation_config = smith_eval.RunEvalConfig( custom_evaluators = [MyStringEvaluator()], ) run_on_dataset( client, dataset_name="", llm_or_chain_factory=construct_chain, evaluation=evaluation_config, ) zrun_on_dataset(zawait arun_on_dataset()r|)NNN)r: __future__rconcurrent.futuresrZ dataclassesrVrloggingr rrtypingrrrrr r r r r Zlangchain_core._apirZ langchain_core.callbacks.managerrZlangchain_core.language_modelsrZlangchain_core.messagesrrZlangchain_core.outputsrrZlangchain_core.runnablesrrrrrrrTZ!langchain_core.tracers.evaluationrrZ langchain_core.tracers.langchainrZlangsmith.clientrZ langsmith.envrr Zlangsmith.evaluationr!r"r#rZlangsmith.run_helpersr$r%Zlangsmith.schemasr&r'r(r)r*Zlangsmith.utilsr+requestsr,Ztyping_extensionsr-Zlangchain.chains.baser.Zlangchain.evaluation.loadingr/Zlangchain.evaluation.schemar0r1r2Zlangchain.smithr3rZlangchain.smith.evaluationrr4r5r^rf getLoggerr7rrcr}r~rr6r=rmrrrrrrrrrrrrrrrrrrrrrrZ dataclassrrBr/rSrYr[Z_RUN_ON_DATASET_DOCSTRINGreplacer;r;r;r<s ,                 I=3 (!%C GE%>C%==C (F(Mk