U hg@sddlmZddlZddlZddlmZmZmZmZm Z m Z m Z m Z m Z mZmZmZddlZddlZddlmZddlmZmZmZmZddlmZmZmZeeZ ddd d d d d ddZ!GdddeeZ"dS)) annotationsN) AnyDictIterableListLiteralMappingOptionalSequenceSetTupleUnioncast) Embeddings) BaseModelField SecretStrroot_validator)from_envget_pydantic_field_namessecret_from_envintzList[Union[List[int], str]]List[List[float]]z List[int]boolzList[Optional[List[float]]]) num_textstokensbatched_embeddingsindices skip_emptyreturnc s&ddt|D}ddt|Dtt|D]J|rJt|dkrJq0||||t|q0g}t|D]|}t|dkr|dqqt|dkr||dqqtfddt|D}tdd|Dd |fd d|Dq|S) NcSsg|]}gqSr .0_r r D/tmp/pip-unpacked-wheel-n8reftby/langchain_openai/embeddings/base.py &sz7_process_batched_chunked_embeddings..cSsg|]}gqSr r r!r r r$r%+srcs,g|]$}tddt|DqS)css|]\}}||VqdSNr )r"valZweightr r r$ JszA_process_batched_chunked_embeddings...)sumzip)r" embedding)inum_tokens_in_batch total_weightr r$r%Is  css|]}|dVqdS)Nr r"r(r r r$r)Tsz6_process_batched_chunked_embeddings..g?csg|] }|qSr r r1) magnituder r$r%Us)rangelenappendr*r+) rrrrrresults embeddingsZ_resultZaverager )r-r2r.r/r$#_process_batched_chunked_embeddingss.       r8c@seZdZUdZedddZded<edddZded<dZd ed <dZ d ed <eZ d ed<ee dddddZ d ed<ede ddddZ d ed<ee ddddZd ed<ee ddddZd ed<dZded<ed ed!dddZd"ed#<ed$e d%d&gdddZd ed'<dZd(ed)<dZd*ed+<d,Zded-<d.Zded/<edd0d1Zd2ed3<dZded4<dZd5ed6<dZd ed7<d8Zd5ed9<eedZd:ed;<d8Zd5ed<<dZ d=ed><dZ!d?ed@<dAZ"dedB<dCZ#dedD<dZ$dEedF<dZ%dEedG<dZ&d5edH<GdIdJdJZ'e(ddKd:d:dLdMdNZ)e(d8dddOdPdPdLdQdRZ*e+d:dSdTdUZ,dVddWdXdYdZZ-dd[dVd d d\d]d^d_Z.dd[dVd d d\d]d`daZ/dmdVd d\dXdcddZ0dndVd d\dXdedfZ1d dgdhdidjZ2d dgdhdkdlZ3dS)oOpenAIEmbeddingsu OpenAI embedding model integration. Setup: Install ``langchain_openai`` and set environment variable ``OPENAI_API_KEY``. .. code-block:: bash pip install -U langchain_openai export OPENAI_API_KEY="your-api-key" Key init args — embedding params: model: str Name of OpenAI model to use. dimensions: Optional[int] = None The number of dimensions the resulting output embeddings should have. Only supported in `text-embedding-3` and later models. Key init args — client params: api_key: Optional[SecretStr] = None OpenAI API key. organization: Optional[str] = None OpenAI organization ID. If not passed in will be read from env var OPENAI_ORG_ID. max_retries: int = 2 Maximum number of retries to make when generating. request_timeout: Optional[Union[float, Tuple[float, float], Any]] = None Timeout for requests to OpenAI completion API See full list of supported init args and their descriptions in the params section. Instantiate: .. code-block:: python from langchain_openai import OpenAIEmbeddings embed = OpenAIEmbeddings( model="text-embedding-3-large" # With the `text-embedding-3` class # of models, you can specify the size # of the embeddings you want returned. # dimensions=1024 ) Embed single text: .. code-block:: python input_text = "The meaning of life is 42" vector = embeddings.embed_query("hello") print(vector[:3]) .. code-block:: python [-0.024603435769677162, -0.007543657906353474, 0.0039630369283258915] Embed multiple texts: .. code-block:: python vectors = embeddings.embed_documents(["hello", "goodbye"]) # Showing only the first 3 coordinates print(len(vectors)) print(vectors[0][:3]) .. code-block:: python 2 [-0.024603435769677162, -0.007543657906353474, 0.0039630369283258915] Async: .. code-block:: python await embed.aembed_query(input_text) print(vector[:3]) # multiple: # await embed.aembed_documents(input_texts) .. code-block:: python [-0.009100092574954033, 0.005071679595857859, -0.0029193938244134188] NT)defaultexcluderclient async_clientztext-embedding-ada-002strmodelz Optional[int] dimensionsz Optional[str] deploymentZOPENAI_API_VERSION)r: api_version)default_factoryaliasopenai_api_versionbase_urlZOPENAI_API_BASE)rDrCopenai_api_baseZOPENAI_API_TYPE)rCopenai_api_typeZ OPENAI_PROXY openai_proxyirembedding_ctx_lengthapi_keyZOPENAI_API_KEYzOptional[SecretStr]openai_api_key organizationZ OPENAI_ORG_IDZOPENAI_ORGANIZATIONopenai_organizationz%Union[Literal['all'], Set[str], None]allowed_specialz4Union[Literal['all'], Set[str], Sequence[str], None]disallowed_speciali chunk_sizer0 max_retriestimeout)r:rDz0Optional[Union[float, Tuple[float, float], Any]]request_timeoutheadersrtiktoken_enabledtiktoken_model_nameFshow_progress_barzDict[str, Any] model_kwargsrzUnion[Mapping[str, str], None]default_headersz!Union[Mapping[str, object], None] default_queryretry_min_secondsretry_max_secondszUnion[Any, None] http_clienthttp_async_clientcheck_embedding_ctx_lengthc@seZdZdZdZdZdS)zOpenAIEmbeddings.Configz'Configuration for this pydantic object.ZforbidTN)__name__ __module__ __qualname____doc__extraZallow_population_by_field_namer r r r$Config srh)pre)valuesrc Cst|}|di}t|D]P}||kr8td|d||krtd|d|d|d||||<q||}|rtd|d ||d<|S) z>Build extra kwargs from additional params that were passed in.rYzFound z supplied twice.z WARNING! z/ is not default parameter. zJ was transferred to model_kwargs. Please confirm that z is what you intended.z Parameters za should be specified explicitly. Instead they were passed in as part of `model_kwargs` parameter.) rgetlist ValueErrorwarningswarnpop intersectionkeys)clsrjZall_required_field_namesrg field_nameZinvalid_model_kwargsr r r$ build_extras"   zOpenAIEmbeddings.build_extra)riZskip_on_failureZ allow_reuserc Cs|ddkrtd|dr(|dnd|d|d|d|d |d |d d }|d r|dsl|dr|d }|d}|d}td|d|d||ds.|d r |ds z ddl}Wn,tk r}ztd|W5d}~XYnX|j|d d|d<d|di}tjf||j|d<|ds|d r|dsz ddl}Wn.tk r}ztd|W5d}~XYnX|j |d d|d<d|di} tj f|| j|d<|S)z?Validate that api key and python package exists in environment.rH)ZazureZazure_adZazureadzEIf you are using Azure, please use the `AzureOpenAIEmbeddings` class.rLNrNrGrTrRrZr[)rKrMrFrSrRrZr[rIr`razwCannot specify 'openai_proxy' if one of 'http_client'/'http_async_client' is already specified. Received: openai_proxy=z http_client=z http_async_client=r<rzRCould not import httpx python package. Please install it with `pip install httpx`.)proxyr=) rmZget_secret_valuerkhttpx ImportErrorZClientopenaiZOpenAIr7Z AsyncClientZ AsyncOpenAI) rsrjZ client_paramsrIr`rarweZ sync_specificZasync_specificr r r$validate_environment*sx          z%OpenAIEmbeddings.validate_environmentrcCs(d|ji|j}|jdk r$|j|d<|S)Nr?r@)r?rYr@)selfparamsr r r$_invocation_paramsks  z#OpenAIEmbeddings._invocation_paramsz List[str]zsz.OpenAIEmbeddings._tokenize..)rOrPZ001  )tqdm)rWr?rVZ transformersrrxrmZfrom_pretrained enumerateencoder3r4rJdecoder5tiktokenZencoding_for_modelKeyErrorZ get_encodingrOrPitemsendswithreplaceZencode_ordinaryrXZ tqdm.autor)r}rrQrrZ model_namer tokenizerr-textZ tokenizedjZ token_chunkZ chunk_textencodingZencoder_kwargstokenr_iterr r r$ _tokenizersb       zOpenAIEmbeddings._tokenize)rQr)renginerQrc s|pj}||\}}}g}|D]R} jjfd|| | |ij} t| ts^| } |dd| dDq$t t ||||j } dddfdd fd d | DS) al Generate length-safe embeddings for a list of texts. This method handles tokenization and embedding generation, respecting the set embedding context length and chunk size. It supports both tiktoken and HuggingFace tokenizer based on the tiktoken_enabled flag. Args: texts (List[str]): A list of texts to embed. engine (str): The engine or model to use for embeddings. chunk_size (Optional[int]): The size of chunks for processing embeddings. Returns: List[List[float]]: A list of embeddings for each input text. inputcss|]}|dVqdSr,Nr r"rr r r$r)sz.dataN List[float]r|csFdkrBjjfddij}t|ts2|}|dddSNrrrr,)r<creater isinstancedict model_dumpZaverage_embedded_cached_empty_embeddingr}r r$empty_embeddings  zBOpenAIEmbeddings._get_len_safe_embeddings..empty_embeddingcsg|]}|dk r|nqSr'r r"rzrr r$r%sz=OpenAIEmbeddings._get_len_safe_embeddings..) rQrr<rrrrrextendr8r4r r}rrrQZ _chunk_sizerrrrr-responser7r rrr}r$_get_len_safe_embeddingss,    z)OpenAIEmbeddings._get_len_safe_embeddingsc s|pj}||\}}}g}|p(j}tdt||D]X} jjfd|| | |ijIdH} t| tsz| } | dd| dDq:t t||||j } dddfd d fd d | DIdHS) a Asynchronously generate length-safe embeddings for a list of texts. This method handles tokenization and asynchronous embedding generation, respecting the set embedding context length and chunk size. It supports both `tiktoken` and HuggingFace `tokenizer` based on the tiktoken_enabled flag. Args: texts (List[str]): A list of texts to embed. engine (str): The engine or model to use for embeddings. chunk_size (Optional[int]): The size of chunks for processing embeddings. Returns: List[List[float]]: A list of embeddings for each input text. rrNcss|]}|dVqdSrr rr r r$r)#sz=OpenAIEmbeddings._aget_len_safe_embeddings..rrr|csLdkrHjjfddijIdH}t|ts8|}|dddSr)r=rrrrrrrr r$r*s   zCOpenAIEmbeddings._aget_len_safe_embeddings..empty_embeddingcs$g|]}|dk r|n IdHqSr'r rrr r$r%5sz>OpenAIEmbeddings._aget_len_safe_embeddings..) rQrr3r4r=rrrrrrr8rrr rr$_aget_len_safe_embeddingss.      z*OpenAIEmbeddings._aget_len_safe_embeddingsrcCst|jsZg}|D]F}|jjfd|i|j}t|ts<|}|dd|dDq|Stt|j }|j ||dS)aMCall out to OpenAI's embedding endpoint for embedding search docs. Args: texts: The list of texts to embed. chunk_size: The chunk size of embeddings. If None, will use the chunk size specified by the class. Returns: List of embeddings, one for each text. rcss|]}|dVqdSrr rr r r$r)Jsz3OpenAIEmbeddings.embed_documents..rr) rbr<rrrrrrr>rArr}rrQr7rrrr r r$embed_documents7s   z OpenAIEmbeddings.embed_documentscs|js`g}|D]L}|jjfd|i|jIdH}t|tsB|}|dd|dDq|Stt|j }|j ||dIdHS)aSCall out to OpenAI's embedding endpoint async for embedding search docs. Args: texts: The list of texts to embed. chunk_size: The chunk size of embeddings. If None, will use the chunk size specified by the class. Returns: List of embeddings, one for each text. rNcss|]}|dVqdSrr rr r r$r)gsz4OpenAIEmbeddings.aembed_documents..rr) rbr=rrrrrrr>rArrr r r$aembed_documentsRs     z!OpenAIEmbeddings.aembed_documentsr)rrcCs||gdS)zCall out to OpenAI's embedding endpoint for embedding query text. Args: text: The text to embed. Returns: Embedding for the text. r)r)r}rr r r$ embed_queryos zOpenAIEmbeddings.embed_querycs||gIdH}|dS)zCall out to OpenAI's embedding endpoint async for embedding query text. Args: text: The text to embed. Returns: Embedding for the text. Nr)r)r}rr7r r r$ aembed_queryzs zOpenAIEmbeddings.aembed_query)r)r)4rcrdrerfrr<__annotations__r=r?r@rArrErGrHrIrJrrLrNrOrPrQrRrTrUrVrWrXrrYrrZr[r]r_r`rarbrhrrur{propertyrrrrrrrrr r r r$r9Zs Q                         @c23 r9)# __future__rloggingrntypingrrrrrrr r r r r rryrZlangchain_core.embeddingsrZlangchain_core.pydantic_v1rrrrZlangchain_core.utilsrrr getLoggerrcloggerr8r9r r r r$s 8  =