U hq@sVddlmZddlZddlZddlZddlmZmZmZm Z m Z m Z m Z m Z mZmZmZmZddlZddlmZddlmZddlmZmZmZddlmZmZmZddl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&dd l'm(Z(e)e*Z+d d d d dZ,d dd ddZ-d"ddddddZ.d dddddZ/d dddddZ0edddd Gd!d d eeZ1dS)#) annotationsN) AnyCallableDictListLiteralMappingOptionalSequenceSetTupleUnioncast) deprecated) Embeddings) BaseModelFieldroot_validator)get_from_dict_or_envget_pydantic_field_namespre_init) AsyncRetryingbefore_sleep_logretryretry_if_exception_typestop_after_attemptwait_exponential) is_openai_v1OpenAIEmbeddingszCallable[[Any], Any]) embeddingsreturncCsnddl}tdt|jtd|j|jdt|jj t|jj Bt|jj Bt|jj Bt|jj BtttjdS)NrTZ multiplierminmaxreraisestopwaitrZ before_sleep)openairr max_retriesrretry_min_secondsretry_max_secondsrerrorTimeoutAPIErrorAPIConnectionErrorRateLimitErrorServiceUnavailableErrorrloggerloggingWARNING)rr)r6I/tmp/pip-unpacked-wheel-9gdii04g/langchain_community/embeddings/openai.py_create_retry_decorator,s(      r8rcsddl}tdt|jtd|j|jdt|jj t|jj Bt|jj Bt|jj Bt|jj Btttjddddfdd }|S) NrTr!r"r%r)funcr csddddfdd }|S)Nrr)argskwargsr cs22z3dHW}||IdHS6tddS)Nzthis is unreachable)AssertionError)r:r;_)async_retryingr9r6r7 wrapped_fbsz7_async_retry_decorator..wrap..wrapped_fr6)r9r?r>)r9r7wrapasz$_async_retry_decorator..wrap)r)rrr*rr+r,rr-r.r/r0r1r2rr3r4r5)rr)rAr6r@r7_async_retry_decoratorGs,      rBFdictbool)response skip_emptyr cCs2tdd|dDr.|s.ddl}|jd|S)Ncss|]}t|ddkVqdS) embeddingr!N)len).0dr6r6r7 nsz"_check_response..datarz&OpenAI API returned an empty embedding)anyr)r-r/)rErFr)r6r6r7_check_responsems rN)rr;r c s>trjjf|St}|dddfdd }|f|S))Use tenacity to retry the embedding call.rr;r csjjf|}t|jdSN)rF)clientcreaterNrFr;rErr6r7_embed_with_retry{sz+embed_with_retry.._embed_with_retry)rrRrSr8)rr;Zretry_decoratorrVr6rUr7embed_with_retryus rWcsFtrjjf|IdHStdddfdd }|f|IdHS)rONrrPcs"jjf|IdH}t|jdSrQ)rRZacreaterNrFrTrUr6r7_async_embed_with_retrysz7async_embed_with_retry.._async_embed_with_retry)r async_clientrSrB)rr;rXr6rUr7async_embed_with_retrys rZz0.0.9z1.0z!langchain_openai.OpenAIEmbeddings)ZsinceZremovalZalternative_importc@s`eZdZUdZedddZded<edddZded<dZd ed <eZ d ed <edd dZ d ed<edddZ d ed<dZ d ed<dZ d ed<dZded<edddZd ed<edddZd ed<eZded<dZded<d Zded!<d"Zded#<edd$dZd%ed&<dZded'<dZd(ed)<dZd ed*<d+Zd(ed,<eed-Zd.ed/<d+Zd(ed0<dZd1ed2<dZd3ed4<d5Z ded6<d7Z!ded8<dZ"d9ed:<Gd;d<d<Z#e$dd=d.d.d>d?d@Z%e&dAdAd>dBdCZ'e(d.dDdEdFZ)ddGdHd dIdJdKdLdMZ*ddGdHd dIdJdKdNdOZ+d\dHdIdJdQdRdSZ,d]dHdIdJdQdTdUZ-d dVdWdXdYZ.d dVdWdZd[Z/dS)^rajOpenAI embedding models. To use, you should have the ``openai`` python package installed, and the environment variable ``OPENAI_API_KEY`` set with your API key or pass it as a named parameter to the constructor. Example: .. code-block:: python from langchain_community.embeddings import OpenAIEmbeddings openai = OpenAIEmbeddings(openai_api_key="my-api-key") In order to use the library with Microsoft Azure endpoints, you need to set the OPENAI_API_TYPE, OPENAI_API_BASE, OPENAI_API_KEY and OPENAI_API_VERSION. The OPENAI_API_TYPE must be set to 'azure' and the others correspond to the properties of your endpoint. In addition, the deployment name must be passed as the model parameter. Example: .. code-block:: python import os os.environ["OPENAI_API_TYPE"] = "azure" os.environ["OPENAI_API_BASE"] = "https://Build extra kwargs from additional params that were passed in.ryzFound z supplied twice.z WARNING! z/ is not default parameter. zJ was transferred to model_kwargs. Please confirm that z is what you intended.z Parameters za should be specified explicitly. Instead they were passed in as part of `model_kwargs` parameter.) rgetlist ValueErrorwarningswarnpop intersectionkeys)clsrZall_required_field_namesr field_nameZinvalid_model_kwargsr6r6r7 build_extras"   zOpenAIEmbeddings.build_extrarc Cst|dd|d<|dp td|d<t|dddd|d<t|d d dd|d <|dd krrd }t|d d|d <nd}t|dd|d|d<|dptdptd|d<z ddl}Wntk rtdYnXtrf|dd krtd|d|d|d|d|d|d|d|dd}| dsF|j f|j |d<| ds~|j f|j |d<n| ds~|j |d<n|S) z?Validate that api key and python package exists in environment.rjZOPENAI_API_KEYrdZOPENAI_API_BASEreZOPENAI_API_TYPE)r[rfZ OPENAI_PROXYZazureZazure_adZazureadz 2023-05-15rpirbZOPENAI_API_VERSIONrlZ OPENAI_ORG_IDZOPENAI_ORGANIZATIONrNTCould not import openai python package. Please install it with `pip install openai`.zfIf you have openai>=1.0.0 installed and are using Azure, please use the `AzureOpenAIEmbeddings` class.rsr*rzr{r~)rirkrcrrr*rzr{r~rRrY)rosgetenvr#r) ImportErrorrrrrZOpenAIrZ AsyncOpenAIZ Embedding)rrZdefault_api_versionr)Z client_paramsr6r6r7validate_environments             z%OpenAIEmbeddings.validate_environment)r c Cstrd|ji|j}n|j|j|j|j|j|j|j|j d|j}|jdkrX|j |d<|j rz ddl }Wnt k rt dYnX|j |j d|_|S)Nr^)r^rsrtrirkZapi_baseZapi_typer`renginerr)httphttps)rr^ryrsrtrjrlrdrerbr_rfr)rproxy)selfZ openai_argsr)r6r6r7_invocation_paramsls4     z#OpenAIEmbeddings._invocation_params)rpz List[str]z Optional[int]zList[List[float]])textsrrpr cCsg}g}|jp|j}|p|j}|jszddlm}Wntk rPtdYnX|j|d} t|D]^\} } | j | dd} t dt | |j D]4} | | | |j }| |}|||| qqfnz ddl}Wntk rtdYnXz||}Wn.tk r.td d }||}YnXt|D]t\} } |jd rZ| d d } |j | |j|jd}t dt ||j D](} ||| | |j || qq8|jrz$ddlm}|t dt ||}Wn&tk rt dt ||}YnXnt dt ||}g}|D]T} t|fd|| | |i|j}t|tsV|}| dd|dDqddt t |D}ddt t |D}t t |D]T} |j!rt || dkr̐q||| || ||| t || qddt t |D}t t |D]} || }t |dkrxt|fddi|j}t|tsf|}|ddd}nt"j#|d|| d}|t"j$%|&|| <q"|S)al Generate length-safe embeddings for a list of texts. This method handles tokenization and embedding generation, respecting the set embedding context length and chunk size. It supports both tiktoken and HuggingFace tokenizer based on the tiktoken_enabled flag. Args: texts (List[str]): A list of texts to embed. engine (str): The engine or model to use for embeddings. chunk_size (Optional[int]): The size of chunks for processing embeddings. Returns: List[List[float]]: A list of embeddings for each input text. r AutoTokenizerzCould not import transformers python package. This is needed in order to for OpenAIEmbeddings without `tiktoken`. Please install it with `pip install transformers`. Zpretrained_model_name_or_pathFZadd_special_tokensNCould not import tiktoken python package. This is needed in order to for OpenAIEmbeddings. Please install it with `pip install tiktoken`.5Warning: model not found. Using cl100k_base encoding. cl100k_base001  textrmro)tqdminputcss|]}|dVqdSrGNr6rIrr6r6r7rKsz.rLcSsg|]}gqSr6r6rIr=r6r6r7 sz=OpenAIEmbeddings._get_len_safe_embeddings..cSsg|]}gqSr6r6rr6r6r7rsr!cSsg|]}gqSr6r6rr6r6r7rsrrGZaxisweights)'rvr^rpru transformersrrfrom_pretrained enumerateencoderangerHrhdecodeappendtiktokenencoding_for_modelKeyErrorr3warning get_encodingendswithreplacermrorwZ tqdm.autorrWr isinstancerCextendrFnpaveragelinalgnormtolist)rrrrptokensindices model_name _chunk_sizer tokenizerir tokenizedj token_chunk chunk_textrencodingr^tokenrZ_iterbatched_embeddingsrEresultsnum_tokens_in_batchr_resultaverage_embeddedrr6r6r7_get_len_safe_embeddingss            z)OpenAIEmbeddings._get_len_safe_embeddingscsLg}g}|jp|j}|p|j}|jszddlm}Wntk rPtdYnX|j|d} t|D]^\} } | j | dd} t dt | |j D]4} | | | |j }| |}|||| qqfnz ddl}Wntk rtdYnXz||}Wn.tk r.td d }||}YnXt|D]t\} } |jd rZ| d d } |j | |j|jd}t dt ||j D](} ||| | |j || qq8g}|p|j}t dt ||D]Z} t|fd|| | |i|jIdH}t|ts|}|dd|dDqddt t |D}ddt t |D}t t |D]6} ||| || ||| t || qbddt t |D}t t |D]} || }t |dkrt|fddi|jIdH}t|ts|}|ddd}ntj|d|| d}|tj !|"|| <q|S)a Asynchronously generate length-safe embeddings for a list of texts. This method handles tokenization and asynchronous embedding generation, respecting the set embedding context length and chunk size. It supports both `tiktoken` and HuggingFace `tokenizer` based on the tiktoken_enabled flag. Args: texts (List[str]): A list of texts to embed. engine (str): The engine or model to use for embeddings. chunk_size (Optional[int]): The size of chunks for processing embeddings. Returns: List[List[float]]: A list of embeddings for each input text. rrzCould not import transformers python package. This is needed in order to for OpenAIEmbeddings without `tiktoken`. Please install it with `pip install transformers`.rFrNrrrrrrrrcss|]}|dVqdSrr6rr6r6r7rKtsz=OpenAIEmbeddings._aget_len_safe_embeddings..rLcSsg|]}gqSr6r6rr6r6r7rvsz>OpenAIEmbeddings._aget_len_safe_embeddings..cSsg|]}gqSr6r6rr6r6r7rwscSsg|]}gqSr6r6rr6r6r7r|srrGr)#rvr^rprurrrrrrrrHrhrrrrrr3rrrrrmrorZrrrCrrrrrr)rrrrprrrrrrrrrrrrrrr^rrrErrrrrrr6r6r7_aget_len_safe_embeddingss              z*OpenAIEmbeddings._aget_len_safe_embeddingsr)rrpr cCstt|j}|j||dS)aMCall out to OpenAI's embedding endpoint for embedding search docs. Args: texts: The list of texts to embed. chunk_size: The chunk size of embeddings. If None, will use the chunk size specified by the class. Returns: List of embeddings, one for each text. r)rr]r_rrrrprr6r6r7embed_documentss z OpenAIEmbeddings.embed_documentscs tt|j}|j||dIdHS)aSCall out to OpenAI's embedding endpoint async for embedding search docs. Args: texts: The list of texts to embed. chunk_size: The chunk size of embeddings. If None, will use the chunk size specified by the class. Returns: List of embeddings, one for each text. rN)rr]r_rrr6r6r7aembed_documentss z!OpenAIEmbeddings.aembed_documentsz List[float])rr cCs||gdS)zCall out to OpenAI's embedding endpoint for embedding query text. Args: text: The text to embed. Returns: Embedding for the text. r)r)rrr6r6r7 embed_querys zOpenAIEmbeddings.embed_querycs||gIdH}|dS)zCall out to OpenAI's embedding endpoint async for embedding query text. Args: text: The text to embed. Returns: Embedding for the text. Nr)r)rrrr6r6r7 aembed_querys zOpenAIEmbeddings.aembed_query)r)r)0rrr__doc__rrR__annotations__rYr^r_rbrdrerfrhrjrlsetrmrorpr*rsrtrurvrwrCryrFrzr{r+r,r~rrrrrpropertyrrrrrrrr6r6r6r7rsb )                 L$z )F)2 __future__rr4rrtypingrrrrrrr r r r r rZnumpyrZlangchain_core._api.deprecationrZlangchain_core.embeddingsrZlangchain_core.pydantic_v1rrrZlangchain_core.utilsrrrZtenacityrrrrrrZ langchain_community.utils.openair getLoggerrr3r8rBrNrWrZrr6r6r6r7s. 8    &