인프런 커뮤니티 질문&답변

올려주신 질문 참조해서 새로 파일을 만들고, jupyter notebook 환경에서도 진행해 보았는데도 오류가 나네요..

구체적으로는
from langchain_chroma import Chroma

database = Chroma.from_documents(

documents=document_list,

embedding=embedding,

collection_name='chroma-tax',

persist_directory="./chroma"

)

다음과 같은 코드를 실행하였을때는

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[8], line 5
      1 from langchain_chroma import Chroma
----> 5 database = Chroma.from_documents(
      6         documents=document_list, 
      7         embedding=embedding, 
      8         collection_name='chroma-tax', 
      9         persist_directory="./chroma"
     10     )
     13 # # 이미 저장된 데이터를 사용할 때 
     14 # database = Chroma(collection_name='chroma-tax', persist_directory="./chroma", embedding_function=embedding)

File ~\LLM_ROOT\L_virutl\lib\site-packages\langchain_chroma\vectorstores.py:1128, in Chroma.from_documents(cls, documents, embedding, ids, collection_name, persist_directory, client_settings, client, collection_metadata, **kwargs)
   1126 texts = [doc.page_content for doc in documents]
   1127 metadatas = [doc.metadata for doc in documents]
-> 1128 return cls.from_texts(
   1129     texts=texts,
   1130     embedding=embedding,
   1131     metadatas=metadatas,
   1132     ids=ids,
   1133     collection_name=collection_name,
   1134     persist_directory=persist_directory,
   1135     client_settings=client_settings,
   1136     client=client,
   1137     collection_metadata=collection_metadata,
   1138     **kwargs,
   1139 )

File ~\LLM_ROOT\L_virutl\lib\site-packages\langchain_chroma\vectorstores.py:1089, in Chroma.from_texts(cls, texts, embedding, metadatas, ids, collection_name, persist_directory, client_settings, client, collection_metadata, **kwargs)
   1083         chroma_collection.add_texts(
   1084             texts=batch[3] if batch[3] else [],
   1085             metadatas=batch[2] if batch[2] else None,  # type: ignore
   1086             ids=batch[0],
   1087         )
   1088 else:
-> 1089     chroma_collection.add_texts(texts=texts, metadatas=metadatas, ids=ids)
   1090 return chroma_collection

File ~\LLM_ROOT\L_virutl\lib\site-packages\langchain_chroma\vectorstores.py:544, in Chroma.add_texts(self, texts, metadatas, ids, **kwargs)
    542             raise ValueError(e.args[0] + "\n\n" + msg)
    543         else:
--> 544             raise e
    545 if empty_ids:
    546     texts_without_metadatas = [texts[j] for j in empty_ids]

File ~\LLM_ROOT\L_virutl\lib\site-packages\langchain_chroma\vectorstores.py:530, in Chroma.add_texts(self, texts, metadatas, ids, **kwargs)
    528 ids_with_metadata = [ids[idx] for idx in non_empty_ids]
    529 try:
--> 530     self._collection.upsert(
    531         metadatas=metadatas,  # type: ignore
    532         embeddings=embeddings_with_metadatas,  # type: ignore
    533         documents=texts_with_metadatas,
    534         ids=ids_with_metadata,
    535     )
    536 except ValueError as e:
    537     if "Expected metadata value to be" in str(e):

File ~\LLM_ROOT\L_virutl\lib\site-packages\chromadb\api\models\Collection.py:302, in Collection.upsert(self, ids, embeddings, metadatas, documents, images, uris)
    281 """Update the embeddings, metadatas or documents for provided ids, or create them if they don't exist.
    282 
    283 Args:
   (...)
    290     None
    291 """
    292 (
    293     ids,
    294     embeddings,
   (...)
    299     ids, embeddings, metadatas, documents, images, uris
    300 )
--> 302 self._client._upsert(
    303     collection_id=self.id,
    304     ids=ids,
    305     embeddings=embeddings,
    306     metadatas=metadatas,
    307     documents=documents,
    308     uris=uris,
    309 )

File ~\LLM_ROOT\L_virutl\lib\site-packages\chromadb\telemetry\opentelemetry\__init__.py:146, in trace_method.<locals>.decorator.<locals>.wrapper(*args, **kwargs)
    144 global tracer, granularity
    145 if trace_granularity < granularity:
--> 146     return f(*args, **kwargs)
    147 if not tracer:
    148     return f(*args, **kwargs)

File ~\LLM_ROOT\L_virutl\lib\site-packages\chromadb\api\segment.py:452, in SegmentAPI._upsert(self, collection_id, ids, embeddings, metadatas, documents, uris)
    450 coll = self._get_collection(collection_id)
    451 self._manager.hint_use_collection(collection_id, t.Operation.UPSERT)
--> 452 validate_batch(
    453     (ids, embeddings, metadatas, documents, uris),
    454     {"max_batch_size": self.get_max_batch_size()},
    455 )
    456 records_to_submit = list(
    457     _records(
    458         t.Operation.UPSERT,
   (...)
    464     )
    465 )
    466 self._validate_embedding_record_set(coll, records_to_submit)

File ~\LLM_ROOT\L_virutl\lib\site-packages\chromadb\api\types.py:571, in validate_batch(batch, limits)
    560 def validate_batch(
    561     batch: Tuple[
    562         IDs,
   (...)
    568     limits: Dict[str, Any],
    569 ) -> None:
    570     if len(batch[0]) > limits["max_batch_size"]:
--> 571         raise ValueError(
    572             f"Batch size {len(batch[0])} exceeds maximum batch size {limits['max_batch_size']}"
    573         )

ValueError: Batch size 220 exceeds maximum batch size 166

이처럼 배치 사이즈 초과 메시지가 뜨고,,

from langchain_chroma import Chroma

# 배치 크기 설정

batch_size = 166

# 문서 리스트를 배치 크기만큼 나누어 처리

for i in range(0, len(document_list), batch_size):

batch_documents = document_list[i:i + batch_size]

database = Chroma.from_documents(documents=batch_documents, embedding=embedding)

추가로 이처럼 배치 사이즈를 줄여 실행하니

이처럼 커널이 죽게 됩니다..

추가로, 환경 변수를 불러오는 설정에서는,

import os

from dotenv import load_dotenv

from langchain_openai import OpenAIEmbeddings

# .env 파일에서 환경변수 불러오기

load_dotenv()

# 환경변수를 불러옴

api_key= os.getenv("OPEN_API_KEY")

# OpenAI에서 제공하는 Embedding Model을 활용해서 chunk를 vector화

embedding = OpenAIEmbeddings(api_key=api_key, model='text-embedding-3-large')

OpenAIEmbeddings안의 매개변수로, api_key가 필요하다 해서 이처럼 전의 OPEN_API_KEY 값을 넣었습니다.

이 부분도 강의와 상이하여 혹시 몰ㄹ ㅏ첨부합니다. 감사합니다!!