core[minor]: Add aload to document loader (#19936)

Add aload to document loader
pull/18013/head
Eugene Yurtsev 2 months ago committed by GitHub
parent 31a641a155
commit d293431e10
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -251,7 +251,7 @@ class WebBaseLoader(BaseLoader):
metadata = _build_metadata(soup, path)
yield Document(page_content=text, metadata=metadata)
def aload(self) -> List[Document]:
def aload(self) -> List[Document]: # type: ignore
"""Load text from the urls in web_path async into Documents."""
results = self.scrape_all(self.web_paths)

@ -28,6 +28,10 @@ class BaseLoader(ABC):
"""Load data into Document objects."""
return list(self.lazy_load())
async def aload(self) -> List[Document]:
"""Load data into Document objects."""
return [document async for document in self.alazy_load()]
def load_and_split(
self, text_splitter: Optional[TextSplitter] = None
) -> List[Document]:

@ -64,3 +64,4 @@ async def test_default_aload() -> None:
docs = loader.load()
assert docs == [Document(page_content="foo"), Document(page_content="bar")]
assert docs == [doc async for doc in loader.alazy_load()]
assert docs == await loader.aload()

Loading…
Cancel
Save