Lint Python notebooks with ruff. (#12677)

The new ruff version fixed the blocking bugs, and I was able to fairly
easily us to a passing state: ruff fixed some issues on its own, I fixed
a handful by hand, and I added a list of narrowly-targeted exclusions
for files that are currently failing ruff rules that we probably should
look into eventually.

I went pretty lenient on the docs / cookbooks rules, allowing dead code
and such things. Perhaps in the future we may want to tighten the rules
further, but this is already a good set of checks that found real issues
and will prevent them going forward.
pull/13371/head
Predrag Gruevski 7 months ago committed by GitHub
parent 344cab0739
commit 2ebd167dba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -3,6 +3,8 @@ import toml
pyproject_toml = toml.load("pyproject.toml")
# Extract the ignore words list (adjust the key as per your TOML structure)
ignore_words_list = pyproject_toml.get("tool", {}).get("codespell", {}).get("ignore-words-list")
ignore_words_list = (
pyproject_toml.get("tool", {}).get("codespell", {}).get("ignore-words-list")
)
print(f"::set-output name=ignore_words_list::{ignore_words_list}")
print(f"::set-output name=ignore_words_list::{ignore_words_list}")

@ -43,10 +43,10 @@ spell_fix:
lint:
poetry run ruff docs templates cookbook
poetry run black docs templates cookbook --diff
poetry run ruff format docs templates cookbook --diff
format format_diff:
poetry run black docs templates cookbook
poetry run ruff format docs templates cookbook
poetry run ruff --select I --fix docs templates cookbook
######################

@ -67,7 +67,6 @@
"llama2_code = ChatOllama(model=\"codellama:7b-instruct\")\n",
"\n",
"# API\n",
"from getpass import getpass\n",
"from langchain.llms import Replicate\n",
"\n",
"# REPLICATE_API_TOKEN = getpass()\n",

@ -102,9 +102,8 @@
"metadata": {},
"outputs": [],
"source": [
"from lxml import html\n",
"from pydantic import BaseModel\n",
"from typing import Any, Optional\n",
"from typing import Any\n",
"from unstructured.partition.pdf import partition_pdf\n",
"\n",
"# Get elements\n",
@ -373,7 +372,6 @@
"metadata": {},
"outputs": [],
"source": [
"from operator import itemgetter\n",
"from langchain.schema.runnable import RunnablePassthrough\n",
"\n",
"# Prompt template\n",

@ -92,9 +92,8 @@
"metadata": {},
"outputs": [],
"source": [
"from lxml import html\n",
"from pydantic import BaseModel\n",
"from typing import Any, Optional\n",
"from typing import Any\n",
"from unstructured.partition.pdf import partition_pdf\n",
"\n",
"# Get elements\n",
@ -224,7 +223,7 @@
"outputs": [],
"source": [
"# Prompt\n",
"prompt_text = \"\"\"You are an assistant tasked with summarizing tables and text. \\ \n",
"prompt_text = \"\"\"You are an assistant tasked with summarizing tables and text. \\\n",
"Give a concise summary of the table or text. Table or text chunk: {element} \"\"\"\n",
"prompt = ChatPromptTemplate.from_template(prompt_text)\n",
"\n",
@ -313,7 +312,7 @@
" # Execute the command and save the output to the defined output file\n",
" /Users/rlm/Desktop/Code/llama.cpp/bin/llava -m ../models/llava-7b/ggml-model-q5_k.gguf --mmproj ../models/llava-7b/mmproj-model-f16.gguf --temp 0.1 -p \"Describe the image in detail. Be specific about graphs, such as bar plots.\" --image \"$img\" > \"$output_file\"\n",
"\n",
"done"
"done\n"
]
},
{
@ -337,7 +336,8 @@
"metadata": {},
"outputs": [],
"source": [
"import os, glob\n",
"import os\n",
"import glob\n",
"\n",
"# Get all .txt file summaries\n",
"file_paths = glob.glob(os.path.expanduser(os.path.join(path, \"*.txt\")))\n",
@ -644,7 +644,6 @@
"metadata": {},
"outputs": [],
"source": [
"from operator import itemgetter\n",
"from langchain.schema.runnable import RunnablePassthrough\n",
"\n",
"# Prompt template\n",

@ -82,10 +82,8 @@
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from lxml import html\n",
"from pydantic import BaseModel\n",
"from typing import Any, Optional\n",
"from typing import Any\n",
"from unstructured.partition.pdf import partition_pdf\n",
"\n",
"# Path to save images\n",
@ -223,7 +221,7 @@
"outputs": [],
"source": [
"# Prompt\n",
"prompt_text = \"\"\"You are an assistant tasked with summarizing tables and text. \\ \n",
"prompt_text = \"\"\"You are an assistant tasked with summarizing tables and text. \\\n",
"Give a concise summary of the table or text. Table or text chunk: {element} \"\"\"\n",
"prompt = ChatPromptTemplate.from_template(prompt_text)\n",
"\n",
@ -312,7 +310,7 @@
" # Execute the command and save the output to the defined output file\n",
" /Users/rlm/Desktop/Code/llama.cpp/bin/llava -m ../models/llava-7b/ggml-model-q5_k.gguf --mmproj ../models/llava-7b/mmproj-model-f16.gguf --temp 0.1 -p \"Describe the image in detail. Be specific about graphs, such as bar plots.\" --image \"$img\" > \"$output_file\"\n",
"\n",
"done"
"done\n"
]
},
{
@ -322,7 +320,8 @@
"metadata": {},
"outputs": [],
"source": [
"import os, glob\n",
"import os\n",
"import glob\n",
"\n",
"# Get all .txt files in the directory\n",
"file_paths = glob.glob(os.path.expanduser(os.path.join(path, \"*.txt\")))\n",
@ -531,7 +530,6 @@
"metadata": {},
"outputs": [],
"source": [
"from operator import itemgetter\n",
"from langchain.schema.runnable import RunnablePassthrough\n",
"\n",
"# Prompt template\n",

@ -63,11 +63,13 @@
"\n",
"# Load\n",
"from langchain.document_loaders import PyPDFLoader\n",
"\n",
"loader = PyPDFLoader(path + \"cpi.pdf\")\n",
"pdf_pages = loader.load()\n",
"\n",
"# Split\n",
"from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
"\n",
"text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=0)\n",
"all_splits_pypdf = text_splitter.split_documents(pdf_pages)\n",
"all_splits_pypdf_texts = [d.page_content for d in all_splits_pypdf]"
@ -132,10 +134,13 @@
"source": [
"from langchain.vectorstores import Chroma\n",
"from langchain.embeddings import OpenAIEmbeddings\n",
"baseline = Chroma.from_texts(texts=all_splits_pypdf_texts,\n",
" collection_name=\"baseline\",\n",
" embedding=OpenAIEmbeddings())\n",
"retriever_baseline=baseline.as_retriever()"
"\n",
"baseline = Chroma.from_texts(\n",
" texts=all_splits_pypdf_texts,\n",
" collection_name=\"baseline\",\n",
" embedding=OpenAIEmbeddings(),\n",
")\n",
"retriever_baseline = baseline.as_retriever()"
]
},
{
@ -160,7 +165,7 @@
"from langchain.schema.output_parser import StrOutputParser\n",
"\n",
"# Prompt\n",
"prompt_text = \"\"\"You are an assistant tasked with summarizing tables and text for retrieval. \\ \n",
"prompt_text = \"\"\"You are an assistant tasked with summarizing tables and text for retrieval. \\\n",
"These summaries will be embedded and used to retrieve the raw text or table elements. \\\n",
"Give a concise summary of the table or text that is well optimized for retrieval. Table or text: {element} \"\"\"\n",
"prompt = ChatPromptTemplate.from_template(prompt_text)\n",
@ -169,7 +174,7 @@
"model = ChatOpenAI(temperature=0, model=\"gpt-4\")\n",
"summarize_chain = {\"element\": lambda x: x} | prompt | model | StrOutputParser()\n",
"\n",
"# Apply to text \n",
"# Apply to text\n",
"text_summaries = summarize_chain.batch(texts, {\"max_concurrency\": 5})\n",
"\n",
"# Apply to tables\n",
@ -192,31 +197,32 @@
"outputs": [],
"source": [
"# Image summary chain\n",
"import os, base64, io\n",
"import os\n",
"import base64\n",
"import io\n",
"from io import BytesIO\n",
"from PIL import Image\n",
"from langchain.schema.messages import HumanMessage\n",
"\n",
"\n",
"def encode_image(image_path):\n",
" ''' Getting the base64 string '''\n",
" \"\"\"Getting the base64 string\"\"\"\n",
" with open(image_path, \"rb\") as image_file:\n",
" return base64.b64encode(image_file.read()).decode('utf-8') \n",
" return base64.b64encode(image_file.read()).decode(\"utf-8\")\n",
"\n",
"\n",
"def image_summarize(img_base64, prompt):\n",
" \"\"\"Image summary\"\"\"\n",
" chat = ChatOpenAI(model=\"gpt-4-vision-preview\", max_tokens=1024)\n",
"\n",
"def image_summarize(img_base64,prompt):\n",
" ''' Image summary '''\n",
" chat = ChatOpenAI(model=\"gpt-4-vision-preview\",\n",
" max_tokens=1024)\n",
" \n",
" msg = chat.invoke(\n",
" [\n",
" HumanMessage(\n",
" content=[\n",
" {\"type\": \"text\", \"text\":prompt},\n",
" {\"type\": \"text\", \"text\": prompt},\n",
" {\n",
" \"type\": \"image_url\",\n",
" \"image_url\": {\n",
" \"url\": f\"data:image/jpeg;base64,{img_base64}\"\n",
" },\n",
" \"image_url\": {\"url\": f\"data:image/jpeg;base64,{img_base64}\"},\n",
" },\n",
" ]\n",
" )\n",
@ -224,6 +230,7 @@
" )\n",
" return msg.content\n",
"\n",
"\n",
"# Store base64 encoded images\n",
"img_base64_list = []\n",
"\n",
@ -231,17 +238,17 @@
"image_summaries = []\n",
"\n",
"# Prompt\n",
"prompt = \"\"\"You are an assistant tasked with summarizing images for retrieval. \\ \n",
"prompt = \"\"\"You are an assistant tasked with summarizing images for retrieval. \\\n",
"These summaries will be embedded and used to retrieve the raw image. \\\n",
"Give a concise summary of the image that is well optimized for retrieval.\"\"\"\n",
"\n",
"# Apply to images\n",
"for img_file in sorted(os.listdir(path)):\n",
" if img_file.endswith('.jpg'):\n",
" if img_file.endswith(\".jpg\"):\n",
" img_path = os.path.join(path, img_file)\n",
" base64_image = encode_image(img_path)\n",
" img_base64_list.append(base64_image)\n",
" image_summaries.append(image_summarize(base64_image,prompt))"
" image_summaries.append(image_summarize(base64_image, prompt))"
]
},
{
@ -267,14 +274,10 @@
"from langchain.schema.document import Document\n",
"from langchain.retrievers.multi_vector import MultiVectorRetriever\n",
"\n",
"def create_multi_vector_retriever(vectorstore, \n",
" text_summaries, \n",
" texts, \n",
" table_summaries, \n",
" tables, \n",
" image_summaries, \n",
" images):\n",
" \n",
"\n",
"def create_multi_vector_retriever(\n",
" vectorstore, text_summaries, texts, table_summaries, tables, image_summaries, images\n",
"):\n",
" # Initialize the storage layer\n",
" store = InMemoryStore()\n",
" id_key = \"doc_id\"\n",
@ -309,18 +312,22 @@
"\n",
" return retriever\n",
"\n",
"\n",
"# The vectorstore to use to index the summaries\n",
"multi_vector_img = Chroma(collection_name=\"multi_vector_img\", \n",
" embedding_function=OpenAIEmbeddings())\n",
"multi_vector_img = Chroma(\n",
" collection_name=\"multi_vector_img\", embedding_function=OpenAIEmbeddings()\n",
")\n",
"\n",
"# Create retriever\n",
"retriever_multi_vector_img = create_multi_vector_retriever(multi_vector_img,\n",
" text_summaries,\n",
" texts,\n",
" table_summaries, \n",
" tables, \n",
" image_summaries, \n",
" img_base64_list)"
"retriever_multi_vector_img = create_multi_vector_retriever(\n",
" multi_vector_img,\n",
" text_summaries,\n",
" texts,\n",
" table_summaries,\n",
" tables,\n",
" image_summaries,\n",
" img_base64_list,\n",
")"
]
},
{
@ -330,10 +337,10 @@
"metadata": {},
"outputs": [],
"source": [
"# Testing on retrieval \n",
"query=\"What percentage of CPI is dedicated to Housing, and how does it compare to the combined percentage of Medical Care, Apparel, and Other Goods and Services?\"\n",
"suffix_for_images=\" Include any pie charts, graphs, or tables.\"\n",
"docs = retriever_multi_vector_img.get_relevant_documents(query+suffix_for_images)"
"# Testing on retrieval\n",
"query = \"What percentage of CPI is dedicated to Housing, and how does it compare to the combined percentage of Medical Care, Apparel, and Other Goods and Services?\"\n",
"suffix_for_images = \" Include any pie charts, graphs, or tables.\"\n",
"docs = retriever_multi_vector_img.get_relevant_documents(query + suffix_for_images)"
]
},
{
@ -357,14 +364,16 @@
],
"source": [
"from IPython.display import display, HTML\n",
"def plt_img_base64(img_base64):\n",
"\n",
"\n",
"def plt_img_base64(img_base64):\n",
" # Create an HTML img tag with the base64 string as the source\n",
" image_html = f'<img src=\"data:image/jpeg;base64,{img_base64}\" />'\n",
" \n",
"\n",
" # Display the image by rendering the HTML\n",
" display(HTML(image_html))\n",
"\n",
"\n",
"plt_img_base64(docs[1])"
]
},
@ -386,17 +395,20 @@
"outputs": [],
"source": [
"# The vectorstore to use to index the summaries\n",
"multi_vector_text = Chroma(collection_name=\"multi_vector_text\", \n",
" embedding_function=OpenAIEmbeddings())\n",
"multi_vector_text = Chroma(\n",
" collection_name=\"multi_vector_text\", embedding_function=OpenAIEmbeddings()\n",
")\n",
"\n",
"# Create retriever\n",
"retriever_multi_vector_img_summary = create_multi_vector_retriever(multi_vector_text,\n",
" text_summaries,\n",
" texts,\n",
" table_summaries, \n",
" tables, \n",
" image_summaries, \n",
" image_summaries)"
"retriever_multi_vector_img_summary = create_multi_vector_retriever(\n",
" multi_vector_text,\n",
" text_summaries,\n",
" texts,\n",
" table_summaries,\n",
" tables,\n",
" image_summaries,\n",
" image_summaries,\n",
")"
]
},
{
@ -418,14 +430,17 @@
"\n",
"# Create chroma w/ multi-modal embeddings\n",
"multimodal_embd = Chroma(\n",
" collection_name=\"multimodal_embd\",\n",
" embedding_function=OpenCLIPEmbeddings()\n",
" collection_name=\"multimodal_embd\", embedding_function=OpenCLIPEmbeddings()\n",
")\n",
"\n",
"# Get image URIs\n",
"image_uris = sorted([os.path.join(path, image_name) \n",
" for image_name in os.listdir(path) \n",
" if image_name.endswith('.jpg')])\n",
"image_uris = sorted(\n",
" [\n",
" os.path.join(path, image_name)\n",
" for image_name in os.listdir(path)\n",
" if image_name.endswith(\".jpg\")\n",
" ]\n",
")\n",
"\n",
"# Add images and documents\n",
"if image_uris:\n",
@ -435,7 +450,7 @@
"if tables:\n",
" multimodal_embd.add_texts(texts=tables)\n",
"\n",
"# Make retriever \n",
"# Make retriever\n",
"retriever_multimodal_embd = multimodal_embd.as_retriever()"
]
},
@ -466,14 +481,14 @@
"\"\"\"\n",
"rag_prompt_text = ChatPromptTemplate.from_template(template)\n",
"\n",
"# Build \n",
"\n",
"# Build\n",
"def text_rag_chain(retriever):\n",
" \n",
" ''' RAG chain '''\n",
" \"\"\"RAG chain\"\"\"\n",
"\n",
" # LLM\n",
" model = ChatOpenAI(temperature=0, model=\"gpt-4\")\n",
" \n",
"\n",
" # RAG pipeline\n",
" chain = (\n",
" {\"context\": retriever, \"question\": RunnablePassthrough()}\n",
@ -500,13 +515,15 @@
"metadata": {},
"outputs": [],
"source": [
"import re \n",
"import re\n",
"from langchain.schema import Document\n",
"from langchain.schema.runnable import RunnableLambda\n",
"\n",
"\n",
"def looks_like_base64(sb):\n",
" \"\"\"Check if the string looks like base64.\"\"\"\n",
" return re.match('^[A-Za-z0-9+/]+[=]{0,2}$', sb) is not None\n",
" return re.match(\"^[A-Za-z0-9+/]+[=]{0,2}$\", sb) is not None\n",
"\n",
"\n",
"def is_image_data(b64data):\n",
" \"\"\"Check if the base64 data is an image by looking at the start of the data.\"\"\"\n",
@ -514,7 +531,7 @@
" b\"\\xFF\\xD8\\xFF\": \"jpg\",\n",
" b\"\\x89\\x50\\x4E\\x47\\x0D\\x0A\\x1A\\x0A\": \"png\",\n",
" b\"\\x47\\x49\\x46\\x38\": \"gif\",\n",
" b\"\\x52\\x49\\x46\\x46\": \"webp\"\n",
" b\"\\x52\\x49\\x46\\x46\": \"webp\",\n",
" }\n",
" try:\n",
" header = base64.b64decode(b64data)[:8] # Decode and get the first 8 bytes\n",
@ -525,6 +542,7 @@
" except Exception:\n",
" return False\n",
"\n",
"\n",
"def split_image_text_types(docs):\n",
" \"\"\"Split base64-encoded images and texts.\"\"\"\n",
" b64_images = []\n",
@ -539,6 +557,7 @@
" texts.append(doc)\n",
" return {\"images\": b64_images, \"texts\": texts}\n",
"\n",
"\n",
"def img_prompt_func(data_dict):\n",
" # Joining the context texts into a single string\n",
" formatted_texts = \"\\n\".join(data_dict[\"context\"][\"texts\"])\n",
@ -550,7 +569,7 @@
" \"type\": \"image_url\",\n",
" \"image_url\": {\n",
" \"url\": f\"data:image/jpeg;base64,{data_dict['context']['images'][0]}\"\n",
" }\n",
" },\n",
" }\n",
" messages.append(image_message)\n",
"\n",
@ -563,22 +582,24 @@
" f\"User-provided question / keywords: {data_dict['question']}\\n\\n\"\n",
" \"Text and / or tables:\\n\"\n",
" f\"{formatted_texts}\"\n",
" )\n",
" ),\n",
" }\n",
" messages.append(text_message)\n",
" return [HumanMessage(content=messages)]\n",
"\n",
"\n",
"def multi_modal_rag_chain(retriever):\n",
" ''' Multi-modal RAG chain '''\n",
" \"\"\"Multi-modal RAG chain\"\"\"\n",
"\n",
" # Multi-modal LLM\n",
" model = ChatOpenAI(temperature=0, \n",
" model=\"gpt-4-vision-preview\", \n",
" max_tokens=1024)\n",
" \n",
" model = ChatOpenAI(temperature=0, model=\"gpt-4-vision-preview\", max_tokens=1024)\n",
"\n",
" # RAG pipeline\n",
" chain = (\n",
" {\"context\": retriever | RunnableLambda(split_image_text_types), \"question\": RunnablePassthrough()}\n",
" {\n",
" \"context\": retriever | RunnableLambda(split_image_text_types),\n",
" \"question\": RunnablePassthrough(),\n",
" }\n",
" | RunnableLambda(img_prompt_func)\n",
" | model\n",
" | StrOutputParser()\n",
@ -603,12 +624,12 @@
"outputs": [],
"source": [
"# RAG chains\n",
"chain_baseline=text_rag_chain(retriever_baseline)\n",
"chain_mv_text=text_rag_chain(retriever_multi_vector_img_summary)\n",
"chain_baseline = text_rag_chain(retriever_baseline)\n",
"chain_mv_text = text_rag_chain(retriever_multi_vector_img_summary)\n",
"\n",
"# Multi-modal RAG chains\n",
"chain_multimodal_mv_img=multi_modal_rag_chain(retriever_multi_vector_img)\n",
"chain_multimodal_embd=multi_modal_rag_chain(retriever_multimodal_embd)"
"chain_multimodal_mv_img = multi_modal_rag_chain(retriever_multi_vector_img)\n",
"chain_multimodal_embd = multi_modal_rag_chain(retriever_multimodal_embd)"
]
},
{
@ -694,7 +715,8 @@
"source": [
"# Read\n",
"import pandas as pd\n",
"eval_set = pd.read_csv(path+'cpi_eval.csv')\n",
"\n",
"eval_set = pd.read_csv(path + \"cpi_eval.csv\")\n",
"eval_set.head(3)"
]
},
@ -715,12 +737,12 @@
"# Populate dataset\n",
"for _, row in eval_set.iterrows():\n",
" # Get Q, A\n",
" q = row['Question']\n",
" a = row['Answer']\n",
" q = row[\"Question\"]\n",
" a = row[\"Answer\"]\n",
" # Use the values in your function\n",
" client.create_example(inputs={\"question\": q}, \n",
" outputs={\"answer\": a}, \n",
" dataset_id=dataset.id)"
" client.create_example(\n",
" inputs={\"question\": q}, outputs={\"answer\": a}, dataset_id=dataset.id\n",
" )"
]
},
{
@ -764,17 +786,22 @@
" evaluators=[\"qa\"],\n",
")\n",
"\n",
"def run_eval(chain,run_name,dataset_name):\n",
"\n",
"def run_eval(chain, run_name, dataset_name):\n",
" _ = client.run_on_dataset(\n",
" dataset_name=dataset_name,\n",
" llm_or_chain_factory=lambda: (lambda x: x[\"question\"]+suffix_for_images) | chain,\n",
" llm_or_chain_factory=lambda: (lambda x: x[\"question\"] + suffix_for_images)\n",
" | chain,\n",
" evaluation=eval_config,\n",
" project_name=run_name,\n",
" )\n",
"\n",
"for chain, run in zip([chain_baseline, chain_mv_text, chain_multimodal_mv_img, chain_multimodal_embd], \n",
" [\"baseline\", \"mv_text\", \"mv_img\", \"mm_embd\"]):\n",
" run_eval(chain, dataset_name+\"-\"+run, dataset_name)"
"\n",
"for chain, run in zip(\n",
" [chain_baseline, chain_mv_text, chain_multimodal_mv_img, chain_multimodal_embd],\n",
" [\"baseline\", \"mv_text\", \"mv_img\", \"mm_embd\"],\n",
"):\n",
" run_eval(chain, dataset_name + \"-\" + run, dataset_name)"
]
}
],

@ -314,7 +314,6 @@
"from langchain.vectorstores import FAISS\n",
"from langchain.docstore import InMemoryDocstore\n",
"from langchain.embeddings import OpenAIEmbeddings\n",
"from langchain.tools.human.tool import HumanInputRun\n",
"\n",
"embeddings_model = OpenAIEmbeddings()\n",
"embedding_size = 1536\n",

@ -29,16 +29,10 @@
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"from collections import deque\n",
"from typing import Dict, List, Optional, Any\n",
"from typing import Optional\n",
"\n",
"from langchain.chains import LLMChain\nfrom langchain.llms import OpenAI\nfrom langchain.prompts import PromptTemplate\n",
"from langchain.llms import OpenAI\n",
"from langchain.embeddings import OpenAIEmbeddings\n",
"from langchain.llms import BaseLLM\n",
"from langchain.schema.vectorstore import VectorStore\n",
"from pydantic import BaseModel, Field\n",
"from langchain.chains.base import Chain\n",
"from langchain_experimental.autonomous_agents import BabyAGI"
]
},

@ -25,16 +25,12 @@
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"from collections import deque\n",
"from typing import Dict, List, Optional, Any\n",
"from typing import Optional\n",
"\n",
"from langchain.chains import LLMChain\nfrom langchain.llms import OpenAI\nfrom langchain.prompts import PromptTemplate\n",
"from langchain.chains import LLMChain\n",
"from langchain.llms import OpenAI\n",
"from langchain.prompts import PromptTemplate\n",
"from langchain.embeddings import OpenAIEmbeddings\n",
"from langchain.llms import BaseLLM\n",
"from langchain.schema.vectorstore import VectorStore\n",
"from pydantic import BaseModel, Field\n",
"from langchain.chains.base import Chain\n",
"from langchain_experimental.autonomous_agents import BabyAGI"
]
},
@ -111,7 +107,9 @@
"outputs": [],
"source": [
"from langchain.agents import ZeroShotAgent, Tool, AgentExecutor\n",
"from langchain.llms import OpenAI\nfrom langchain.utilities import SerpAPIWrapper\nfrom langchain.chains import LLMChain\n",
"from langchain.llms import OpenAI\n",
"from langchain.utilities import SerpAPIWrapper\n",
"from langchain.chains import LLMChain\n",
"\n",
"todo_prompt = PromptTemplate.from_template(\n",
" \"You are a planner who is an expert at coming up with a todo list for a given objective. Come up with a todo list for this objective: {objective}\"\n",

@ -177,7 +177,7 @@
" try:\n",
" loader = TextLoader(os.path.join(dirpath, file), encoding=\"utf-8\")\n",
" docs.extend(loader.load_and_split())\n",
" except Exception as e:\n",
" except Exception:\n",
" pass\n",
"print(f\"{len(docs)}\")"
]

@ -33,14 +33,14 @@
"outputs": [],
"source": [
"from langchain.agents import (\n",
" Tool,\n",
" AgentExecutor,\n",
" LLMSingleActionAgent,\n",
" AgentOutputParser,\n",
")\n",
"from langchain.prompts import StringPromptTemplate\n",
"from langchain.llms import OpenAI\nfrom langchain.utilities import SerpAPIWrapper\nfrom langchain.chains import LLMChain\n",
"from typing import List, Union\n",
"from langchain.llms import OpenAI\n",
"from langchain.chains import LLMChain\n",
"from typing import Union\n",
"from langchain.schema import AgentAction, AgentFinish\n",
"from langchain.agents.agent_toolkits import NLAToolkit\n",
"from langchain.tools.plugin import AIPlugin\n",

@ -57,14 +57,14 @@
"outputs": [],
"source": [
"from langchain.agents import (\n",
" Tool,\n",
" AgentExecutor,\n",
" LLMSingleActionAgent,\n",
" AgentOutputParser,\n",
")\n",
"from langchain.prompts import StringPromptTemplate\n",
"from langchain.llms import OpenAI\nfrom langchain.utilities import SerpAPIWrapper\nfrom langchain.chains import LLMChain\n",
"from typing import List, Union\n",
"from langchain.llms import OpenAI\n",
"from langchain.chains import LLMChain\n",
"from typing import Union\n",
"from langchain.schema import AgentAction, AgentFinish\n",
"from langchain.agents.agent_toolkits import NLAToolkit\n",
"from langchain.tools.plugin import AIPlugin\n",

@ -50,15 +50,13 @@
"source": [
"import os\n",
"import getpass\n",
"from langchain.document_loaders import PyPDFLoader, TextLoader\n",
"from langchain.embeddings.openai import OpenAIEmbeddings\n",
"from langchain.text_splitter import (\n",
" RecursiveCharacterTextSplitter,\n",
" CharacterTextSplitter,\n",
")\n",
"from langchain.vectorstores import DeepLake\n",
"from langchain.chains import ConversationalRetrievalChain, RetrievalQA\n",
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.chains import RetrievalQA\n",
"from langchain.llms import OpenAI\n",
"\n",
"os.environ[\"OPENAI_API_KEY\"] = getpass.getpass(\"OpenAI API Key:\")\n",

@ -112,7 +112,6 @@
"metadata": {},
"outputs": [],
"source": [
"from langchain.chains.elasticsearch_database.prompts import DEFAULT_DSL_TEMPLATE\n",
"from langchain.prompts.prompt import PromptTemplate\n",
"\n",
"PROMPT_TEMPLATE = \"\"\"Given an input question, create a syntactically correct Elasticsearch query to run. Unless the user specifies in their question a specific number of examples they wish to obtain, always limit your query to at most {top_k} results. You can order the results by a relevant column to return the most interesting examples in the database.\n",

@ -56,7 +56,8 @@
"source": [
"import os\n",
"\n",
"os.environ[\"SERPER_API_KEY\"] = \"\"os.environ[\"OPENAI_API_KEY\"] = \"\""
"os.environ[\"SERPER_API_KEY\"] = \"\"\n",
"os.environ[\"OPENAI_API_KEY\"] = \"\""
]
},
{
@ -66,17 +67,12 @@
"metadata": {},
"outputs": [],
"source": [
"import re\n",
"\n",
"import numpy as np\n",
"\n",
"from langchain.schema import BaseRetriever\n",
"from langchain.callbacks.manager import (\n",
" AsyncCallbackManagerForRetrieverRun,\n",
" CallbackManagerForRetrieverRun,\n",
")\n",
"from langchain.utilities import GoogleSerperAPIWrapper\n",
"from langchain.embeddings import OpenAIEmbeddings\n",
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.llms import OpenAI\n",
"from langchain.schema import Document\n",

@ -27,16 +27,11 @@
"metadata": {},
"outputs": [],
"source": [
"import gymnasium as gym\n",
"import inspect\n",
"import tenacity\n",
"\n",
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.schema import (\n",
" AIMessage,\n",
" HumanMessage,\n",
" SystemMessage,\n",
" BaseMessage,\n",
")\n",
"from langchain.output_parsers import RegexParser"
]
@ -131,7 +126,7 @@
" ):\n",
" with attempt:\n",
" action = self._act()\n",
" except tenacity.RetryError as e:\n",
" except tenacity.RetryError:\n",
" action = self.random_action()\n",
" return action"
]

@ -56,7 +56,9 @@
"metadata": {},
"outputs": [],
"source": [
"from langchain.llms import OpenAI\nfrom langchain.chains import LLMChain\nfrom langchain.prompts import PromptTemplate\n",
"from langchain.llms import OpenAI\n",
"from langchain.chains import LLMChain\n",
"from langchain.prompts import PromptTemplate\n",
"from langchain.memory import ConversationBufferWindowMemory"
]
},
@ -152,13 +154,13 @@
" for j in range(max_iters):\n",
" print(f\"(Step {j+1}/{max_iters})\")\n",
" print(f\"Assistant: {output}\")\n",
" print(f\"Human: \")\n",
" print(\"Human: \")\n",
" human_input = input()\n",
" if any(phrase in human_input.lower() for phrase in key_phrases):\n",
" break\n",
" output = chain.predict(human_input=human_input)\n",
" if success_phrase in human_input.lower():\n",
" print(f\"You succeeded! Thanks for playing!\")\n",
" print(\"You succeeded! Thanks for playing!\")\n",
" return\n",
" meta_chain = initialize_meta_chain()\n",
" meta_output = meta_chain.predict(chat_history=get_chat_history(chain.memory))\n",
@ -166,7 +168,7 @@
" instructions = get_new_instructions(meta_output)\n",
" print(f\"New Instructions: {instructions}\")\n",
" print(\"\\n\" + \"#\" * 80 + \"\\n\")\n",
" print(f\"You failed! Thanks for playing!\")"
" print(\"You failed! Thanks for playing!\")"
]
},
{

@ -115,7 +115,7 @@
"metadata": {},
"outputs": [],
"source": [
"# Folder with pdf and extracted images \n",
"# Folder with pdf and extracted images\n",
"path = \"/Users/rlm/Desktop/photos/\""
]
},
@ -128,9 +128,10 @@
"source": [
"# Extract images, tables, and chunk text\n",
"from unstructured.partition.pdf import partition_pdf\n",
"\n",
"raw_pdf_elements = partition_pdf(\n",
" filename=path + \"photos.pdf\",\n",
" extract_images_in_pdf=True, \n",
" extract_images_in_pdf=True,\n",
" infer_table_structure=True,\n",
" chunking_strategy=\"by_title\",\n",
" max_characters=4000,\n",
@ -191,14 +192,17 @@
"\n",
"# Create chroma\n",
"vectorstore = Chroma(\n",
" collection_name=\"mm_rag_clip_photos\",\n",
" embedding_function=OpenCLIPEmbeddings()\n",
" collection_name=\"mm_rag_clip_photos\", embedding_function=OpenCLIPEmbeddings()\n",
")\n",
"\n",
"# Get image URIs with .jpg extension only\n",
"image_uris = sorted([os.path.join(path, image_name) \n",
" for image_name in os.listdir(path) \n",
" if image_name.endswith('.jpg')])\n",
"image_uris = sorted(\n",
" [\n",
" os.path.join(path, image_name)\n",
" for image_name in os.listdir(path)\n",
" if image_name.endswith(\".jpg\")\n",
" ]\n",
")\n",
"\n",
"# Add images\n",
"vectorstore.add_images(uris=image_uris)\n",
@ -206,7 +210,7 @@
"# Add documents\n",
"vectorstore.add_texts(texts=texts)\n",
"\n",
"# Make retriever \n",
"# Make retriever\n",
"retriever = vectorstore.as_retriever()"
]
},
@ -235,6 +239,7 @@
"from io import BytesIO\n",
"from PIL import Image\n",
"\n",
"\n",
"def resize_base64_image(base64_string, size=(128, 128)):\n",
" \"\"\"\n",
" Resize an image encoded as a Base64 string.\n",
@ -258,30 +263,31 @@
" resized_img.save(buffered, format=img.format)\n",
"\n",
" # Encode the resized image to Base64\n",
" return base64.b64encode(buffered.getvalue()).decode('utf-8')\n",
" return base64.b64encode(buffered.getvalue()).decode(\"utf-8\")\n",
"\n",
"\n",
"def is_base64(s):\n",
" ''' Check if a string is Base64 encoded '''\n",
" \"\"\"Check if a string is Base64 encoded\"\"\"\n",
" try:\n",
" return base64.b64encode(base64.b64decode(s)) == s.encode()\n",
" except Exception:\n",
" return False\n",
" \n",
"\n",
"\n",
"def split_image_text_types(docs):\n",
" ''' Split numpy array images and texts '''\n",
" \"\"\"Split numpy array images and texts\"\"\"\n",
" images = []\n",
" text = []\n",
" for doc in docs:\n",
" doc = doc.page_content # Extract Document contents \n",
" doc = doc.page_content # Extract Document contents\n",
" if is_base64(doc):\n",
" # Resize image to avoid OAI server error\n",
" images.append(resize_base64_image(doc, size=(250, 250))) # base64 encoded str \n",
" images.append(\n",
" resize_base64_image(doc, size=(250, 250))\n",
" ) # base64 encoded str\n",
" else:\n",
" text.append(doc) \n",
" return {\n",
" \"images\": images,\n",
" \"texts\": text\n",
" }"
" text.append(doc)\n",
" return {\"images\": images, \"texts\": text}"
]
},
{
@ -311,6 +317,7 @@
"from langchain.schema.runnable import RunnablePassthrough, RunnableLambda\n",
"from langchain.schema.messages import HumanMessage, SystemMessage\n",
"\n",
"\n",
"def prompt_func(data_dict):\n",
" # Joining the context texts into a single string\n",
" formatted_texts = \"\\n\".join(data_dict[\"context\"][\"texts\"])\n",
@ -322,7 +329,7 @@
" \"type\": \"image_url\",\n",
" \"image_url\": {\n",
" \"url\": f\"data:image/jpeg;base64,{data_dict['context']['images'][0]}\"\n",
" }\n",
" },\n",
" }\n",
" messages.append(image_message)\n",
"\n",
@ -342,17 +349,21 @@
" f\"User-provided keywords: {data_dict['question']}\\n\\n\"\n",
" \"Text and / or tables:\\n\"\n",
" f\"{formatted_texts}\"\n",
" )\n",
" ),\n",
" }\n",
" messages.append(text_message)\n",
"\n",
" return [HumanMessage(content=messages)]\n",
" \n",
"\n",
"\n",
"model = ChatOpenAI(temperature=0, model=\"gpt-4-vision-preview\", max_tokens=1024)\n",
"\n",
"# RAG pipeline\n",
"chain = (\n",
" {\"context\": retriever | RunnableLambda(split_image_text_types), \"question\": RunnablePassthrough()}\n",
" {\n",
" \"context\": retriever | RunnableLambda(split_image_text_types),\n",
" \"question\": RunnablePassthrough(),\n",
" }\n",
" | RunnableLambda(prompt_func)\n",
" | model\n",
" | StrOutputParser()\n",
@ -412,15 +423,16 @@
"source": [
"from IPython.display import display, HTML\n",
"\n",
"def plt_img_base64(img_base64):\n",
"\n",
"def plt_img_base64(img_base64):\n",
" # Create an HTML img tag with the base64 string as the source\n",
" image_html = f'<img src=\"data:image/jpeg;base64,{img_base64}\" />'\n",
" \n",
"\n",
" # Display the image by rendering the HTML\n",
" display(HTML(image_html))\n",
"\n",
"docs = retriever.get_relevant_documents(\"Woman with children\",k=10)\n",
"\n",
"docs = retriever.get_relevant_documents(\"Woman with children\", k=10)\n",
"for doc in docs:\n",
" if is_base64(doc.page_content):\n",
" plt_img_base64(doc.page_content)\n",
@ -446,9 +458,7 @@
}
],
"source": [
"chain.invoke(\n",
" \"Woman with children\"\n",
")"
"chain.invoke(\"Woman with children\")"
]
},
{

@ -26,13 +26,11 @@
"metadata": {},
"outputs": [],
"source": [
"from typing import List, Dict, Callable\n",
"from typing import List, Callable\n",
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.schema import (\n",
" AIMessage,\n",
" HumanMessage,\n",
" SystemMessage,\n",
" BaseMessage,\n",
")"
]
},

@ -30,23 +30,17 @@
"from collections import OrderedDict\n",
"import functools\n",
"import random\n",
"import re\n",
"import tenacity\n",
"from typing import List, Dict, Callable\n",
"from typing import List, Callable\n",
"\n",
"from langchain.prompts import (\n",
" ChatPromptTemplate,\n",
" HumanMessagePromptTemplate,\n",
" PromptTemplate,\n",
")\n",
"from langchain.chains import LLMChain\n",
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.output_parsers import RegexParser\n",
"from langchain.schema import (\n",
" AIMessage,\n",
" HumanMessage,\n",
" SystemMessage,\n",
" BaseMessage,\n",
")"
]
},

@ -25,16 +25,13 @@
"outputs": [],
"source": [
"from langchain.prompts import PromptTemplate\n",
"import re\n",
"import tenacity\n",
"from typing import List, Dict, Callable\n",
"from typing import List, Callable\n",
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.output_parsers import RegexParser\n",
"from langchain.schema import (\n",
" AIMessage,\n",
" HumanMessage,\n",
" SystemMessage,\n",
" BaseMessage,\n",
")"
]
},

@ -29,12 +29,11 @@
"source": [
"from os import environ\n",
"import getpass\n",
"from typing import Dict, Any\n",
"from langchain.llms import OpenAI\n",
"from langchain.utilities import SQLDatabase\n",
"from langchain.chains import LLMChain\n",
"from langchain_experimental.sql.vector_sql import VectorSQLDatabaseChain\n",
"from sqlalchemy import create_engine, Column, MetaData\n",
"from sqlalchemy import create_engine, MetaData\n",
"from langchain.prompts import PromptTemplate\n",
"\n",
"\n",

@ -146,7 +146,7 @@
" ):\n",
" with attempt:\n",
" action = self._act()\n",
" except tenacity.RetryError as e:\n",
" except tenacity.RetryError:\n",
" action = self.random_action()\n",
" return action"
]

@ -82,7 +82,9 @@
"secret_access_key = \"your bos access sk\"\n",
"\n",
"# create BceClientConfiguration\n",
"config = BceClientConfiguration(credentials=BceCredentials(access_key_id, secret_access_key), endpoint = bos_host)\n",
"config = BceClientConfiguration(\n",
" credentials=BceCredentials(access_key_id, secret_access_key), endpoint=bos_host\n",
")\n",
"\n",
"loader = BaiduBOSDirectoryLoader(conf=config, bucket=\"llm-test\", prefix=\"llm/\")\n",
"documents = loader.load()\n",
@ -109,10 +111,14 @@
"embeddings.client = sentence_transformers.SentenceTransformer(embeddings.model_name)\n",
"\n",
"db = BESVectorStore.from_documents(\n",
" documents=split_docs, embedding=embeddings, bes_url=\"your bes url\", index_name='test-index', vector_query_field='vector'\n",
" )\n",
" documents=split_docs,\n",
" embedding=embeddings,\n",
" bes_url=\"your bes url\",\n",
" index_name=\"test-index\",\n",
" vector_query_field=\"vector\",\n",
")\n",
"\n",
"db.client.indices.refresh(index='test-index')\n",
"db.client.indices.refresh(index=\"test-index\")\n",
"retriever = db.as_retriever()"
]
},
@ -130,8 +136,15 @@
"metadata": {},
"outputs": [],
"source": [
"llm = QianfanLLMEndpoint(model=\"ERNIE-Bot\", qianfan_ak='your qianfan ak', qianfan_sk='your qianfan sk', streaming=True)\n",
"qa = RetrievalQA.from_chain_type(llm=llm, chain_type=\"refine\", retriever=retriever, return_source_documents=True)\n",
"llm = QianfanLLMEndpoint(\n",
" model=\"ERNIE-Bot\",\n",
" qianfan_ak=\"your qianfan ak\",\n",
" qianfan_sk=\"your qianfan sk\",\n",
" streaming=True,\n",
")\n",
"qa = RetrievalQA.from_chain_type(\n",
" llm=llm, chain_type=\"refine\", retriever=retriever, return_source_documents=True\n",
")\n",
"\n",
"query = \"什么是张量?\"\n",
"print(qa.run(query))"

@ -87,7 +87,6 @@
"outputs": [],
"source": [
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.prompts import ChatPromptTemplate\n",
"from langchain.schema.output_parser import StrOutputParser"
]
},

@ -31,12 +31,10 @@
"metadata": {},
"outputs": [],
"source": [
"from operator import itemgetter\n",
"\n",
"from langchain.prompts import ChatPromptTemplate\n",
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.schema.output_parser import StrOutputParser\n",
"from langchain.schema.runnable import RunnablePassthrough, RunnableLambda\n",
"from langchain.schema.runnable import RunnablePassthrough\n",
"from langchain.utilities import DuckDuckGoSearchAPIWrapper"
]
},

@ -19,10 +19,8 @@
"source": [
"from langchain.prompts import PromptTemplate\n",
"from langchain.schema.prompt import PromptValue\n",
"from langchain.schema.messages import BaseMessage\n",
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.schema.output_parser import StrOutputParser\n",
"from typing import Union, Sequence"
"from langchain.schema.output_parser import StrOutputParser"
]
},
{

@ -1084,7 +1084,6 @@
"outputs": [],
"source": [
"from langchain.embeddings import OpenAIEmbeddings\n",
"from langchain.schema import Document\n",
"from langchain.vectorstores import ElasticsearchStore\n",
"\n",
"embeddings = OpenAIEmbeddings()"

@ -118,7 +118,7 @@
" try:\n",
" loader = TextLoader(os.path.join(dirpath, file), encoding=\"utf-8\")\n",
" docs.extend(loader.load_and_split())\n",
" except Exception as e:\n",
" except Exception:\n",
" pass"
]
},

@ -22,17 +22,13 @@
"metadata": {},
"outputs": [],
"source": [
"from typing import List, Dict, Callable\n",
"from langchain.chains import ConversationChain\n",
"from typing import List, Callable\n",
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.llms import OpenAI\n",
"from langchain.memory import ConversationBufferMemory\n",
"from langchain.prompts.prompt import PromptTemplate\n",
"from langchain.schema import (\n",
" AIMessage,\n",
" HumanMessage,\n",
" SystemMessage,\n",
" BaseMessage,\n",
")"
]
},
@ -49,7 +45,6 @@
"metadata": {},
"outputs": [],
"source": [
"from langchain.agents import Tool\n",
"from langchain.agents import initialize_agent\n",
"from langchain.agents import AgentType\n",
"from langchain.agents import load_tools"

@ -22,7 +22,7 @@
"metadata": {},
"outputs": [],
"source": [
"from typing import List, Dict, Callable\n",
"from typing import List, Callable\n",
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.schema import (\n",
" HumanMessage,\n",

@ -396,7 +396,7 @@
" AgentOutputParser,\n",
")\n",
"from langchain.prompts import StringPromptTemplate\n",
"from langchain.llms import OpenAI\nfrom langchain.chains import LLMChain\n",
"from langchain.chains import LLMChain\n",
"from typing import List, Union\n",
"from langchain.schema import AgentAction, AgentFinish\n",
"import re"

@ -20,8 +20,6 @@
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.prompts import (\n",
" ChatPromptTemplate,\n",
" SystemMessagePromptTemplate,\n",
" HumanMessagePromptTemplate,\n",
")\n",
"from langchain.schema.output_parser import StrOutputParser\n",
"from langchain_experimental.utilities import PythonREPL"

@ -69,7 +69,7 @@
"metadata": {},
"outputs": [],
"source": [
"from langchain.schema.runnable import RunnableMap, RunnablePassthrough\n",
"from langchain.schema.runnable import RunnablePassthrough\n",
"\n",
"prompt1 = ChatPromptTemplate.from_template(\n",
" \"generate a {attribute} color. Return the name of the color and nothing else:\"\n",

@ -33,7 +33,7 @@
}
],
"source": [
"from langchain.evaluation import JsonValidityEvaluator, load_evaluator\n",
"from langchain.evaluation import JsonValidityEvaluator\n",
"\n",
"evaluator = JsonValidityEvaluator()\n",
"# Equivalently\n",

@ -1,305 +1,304 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "6e5ea1a1-7e74-459b-bf14-688f87d09124",
"metadata": {
"tags": []
},
"source": [
"# Agent Trajectory\n",
"[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain/blob/master/docs/docs/guides/evaluation/trajectory/trajectory_eval.ipynb)\n",
"\n",
"Agents can be difficult to holistically evaluate due to the breadth of actions and generation they can make. We recommend using multiple evaluation techniques appropriate to your use case. One way to evaluate an agent is to look at the whole trajectory of actions taken along with their responses.\n",
"\n",
"Evaluators that do this can implement the `AgentTrajectoryEvaluator` interface. This walkthrough will show how to use the `trajectory` evaluator to grade an OpenAI functions agent.\n",
"\n",
"For more information, check out the reference docs for the [TrajectoryEvalChain](https://api.python.langchain.com/en/latest/evaluation/langchain.evaluation.agents.trajectory_eval_chain.TrajectoryEvalChain.html#langchain.evaluation.agents.trajectory_eval_chain.TrajectoryEvalChain) for more info."
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "149402da-5212-43e2-b7c0-a701727f5293",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"from langchain.evaluation import load_evaluator\n",
"\n",
"evaluator = load_evaluator(\"trajectory\")"
]
},
{
"cell_type": "markdown",
"id": "b1c64c1a",
"metadata": {},
"source": [
"## Methods\n",
"\n",
"\n",
"The Agent Trajectory Evaluators are used with the [evaluate_agent_trajectory](https://api.python.langchain.com/en/latest/evaluation/langchain.evaluation.agents.trajectory_eval_chain.TrajectoryEvalChain.html#langchain.evaluation.agents.trajectory_eval_chain.TrajectoryEvalChain.evaluate_agent_trajectory) (and async [aevaluate_agent_trajectory](https://api.python.langchain.com/en/latest/evaluation/langchain.evaluation.agents.trajectory_eval_chain.TrajectoryEvalChain.html#langchain.evaluation.agents.trajectory_eval_chain.TrajectoryEvalChain.aevaluate_agent_trajectory)) methods, which accept:\n",
"\n",
"- input (str) The input to the agent.\n",
"- prediction (str) The final predicted response.\n",
"- agent_trajectory (List[Tuple[AgentAction, str]]) The intermediate steps forming the agent trajectory\n",
"\n",
"They return a dictionary with the following values:\n",
"- score: Float from 0 to 1, where 1 would mean \"most effective\" and 0 would mean \"least effective\"\n",
"- reasoning: String \"chain of thought reasoning\" from the LLM generated prior to creating the score"
]
},
{
"cell_type": "markdown",
"id": "e733562c-4c17-4942-9647-acfc5ebfaca2",
"metadata": {},
"source": [
"## Capturing Trajectory\n",
"\n",
"The easiest way to return an agent's trajectory (without using tracing callbacks like those in LangSmith) for evaluation is to initialize the agent with `return_intermediate_steps=True`.\n",
"\n",
"Below, create an example agent we will call to evaluate."
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "451cb0cb-6f42-4abd-aa6d-fb871fce034d",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import os\n",
"import subprocess\n",
"\n",
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.tools import tool\n",
"from langchain.agents import AgentType, initialize_agent\n",
"\n",
"from pydantic import HttpUrl\n",
"from urllib.parse import urlparse\n",
"\n",
"\n",
"@tool\n",
"def ping(url: HttpUrl, return_error: bool) -> str:\n",
" \"\"\"Ping the fully specified url. Must include https:// in the url.\"\"\"\n",
" hostname = urlparse(str(url)).netloc\n",
" completed_process = subprocess.run(\n",
" [\"ping\", \"-c\", \"1\", hostname], capture_output=True, text=True\n",
" )\n",
" output = completed_process.stdout\n",
" if return_error and completed_process.returncode != 0:\n",
" return completed_process.stderr\n",
" return output\n",
"\n",
"\n",
"@tool\n",
"def trace_route(url: HttpUrl, return_error: bool) -> str:\n",
" \"\"\"Trace the route to the specified url. Must include https:// in the url.\"\"\"\n",
" hostname = urlparse(str(url)).netloc\n",
" completed_process = subprocess.run(\n",
" [\"traceroute\", hostname], capture_output=True, text=True\n",
" )\n",
" output = completed_process.stdout\n",
" if return_error and completed_process.returncode != 0:\n",
" return completed_process.stderr\n",
" return output\n",
"\n",
"\n",
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-0613\", temperature=0)\n",
"agent = initialize_agent(\n",
" llm=llm,\n",
" tools=[ping, trace_route],\n",
" agent=AgentType.OPENAI_MULTI_FUNCTIONS,\n",
" return_intermediate_steps=True, # IMPORTANT!\n",
")\n",
"\n",
"result = agent(\"What's the latency like for https://langchain.com?\")"
]
},
{
"cell_type": "markdown",
"id": "2df34eed-45a5-4f91-88d3-9aa55f28391a",
"metadata": {
"tags": []
},
"source": [
"## Evaluate Trajectory\n",
"\n",
"Pass the input, trajectory, and pass to the [evaluate_agent_trajectory](https://api.python.langchain.com/en/latest/evaluation/langchain.evaluation.schema.AgentTrajectoryEvaluator.html#langchain.evaluation.schema.AgentTrajectoryEvaluator.evaluate_agent_trajectory) method."
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "8d2c8703-98ed-4068-8a8b-393f0f1f64ea",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"{'score': 1.0,\n",
" 'reasoning': \"i. The final answer is helpful. It directly answers the user's question about the latency for the website https://langchain.com.\\n\\nii. The AI language model uses a logical sequence of tools to answer the question. It uses the 'ping' tool to measure the latency of the website, which is the correct tool for this task.\\n\\niii. The AI language model uses the tool in a helpful way. It inputs the URL into the 'ping' tool and correctly interprets the output to provide the latency in milliseconds.\\n\\niv. The AI language model does not use too many steps to answer the question. It only uses one step, which is appropriate for this type of question.\\n\\nv. The appropriate tool is used to answer the question. The 'ping' tool is the correct tool to measure website latency.\\n\\nGiven these considerations, the AI language model's performance is excellent. It uses the correct tool, interprets the output correctly, and provides a helpful and direct answer to the user's question.\"}"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"evaluation_result = evaluator.evaluate_agent_trajectory(\n",
" prediction=result[\"output\"],\n",
" input=result[\"input\"],\n",
" agent_trajectory=result[\"intermediate_steps\"],\n",
")\n",
"evaluation_result"
]
},
{
"cell_type": "markdown",
"id": "fc5467c1-ea92-405f-949a-3011388fa9ee",
"metadata": {},
"source": [
"## Configuring the Evaluation LLM\n",
"\n",
"If you don't select an LLM to use for evaluation, the [load_evaluator](https://api.python.langchain.com/en/latest/evaluation/langchain.evaluation.loading.load_evaluator.html#langchain.evaluation.loading.load_evaluator) function will use `gpt-4` to power the evaluation chain. You can select any chat model for the agent trajectory evaluator as below."
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "1f6318f3-642a-4766-bc7a-f91239795ee7",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# %pip install anthropic\n",
"# ANTHROPIC_API_KEY=<YOUR ANTHROPIC API KEY>"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "b2852289-5df9-402e-95b5-7efebf0fc943",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"from langchain.chat_models import ChatAnthropic\n",
"\n",
"eval_llm = ChatAnthropic(temperature=0)\n",
"evaluator = load_evaluator(\"trajectory\", llm=eval_llm)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "ff72d21a-93b9-4c2f-8613-733d9c9330d7",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"{'score': 1.0,\n",
" 'reasoning': \"Here is my detailed evaluation of the AI's response:\\n\\ni. The final answer is helpful, as it directly provides the latency measurement for the requested website.\\n\\nii. The sequence of using the ping tool to measure latency is logical for this question.\\n\\niii. The ping tool is used in a helpful way, with the website URL provided as input and the output latency measurement extracted.\\n\\niv. Only one step is used, which is appropriate for simply measuring latency. More steps are not needed.\\n\\nv. The ping tool is an appropriate choice to measure latency. \\n\\nIn summary, the AI uses an optimal single step approach with the right tool and extracts the needed output. The final answer directly answers the question in a helpful way.\\n\\nOverall\"}"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"evaluation_result = evaluator.evaluate_agent_trajectory(\n",
" prediction=result[\"output\"],\n",
" input=result[\"input\"],\n",
" agent_trajectory=result[\"intermediate_steps\"],\n",
")\n",
"evaluation_result"
]
},
{
"cell_type": "markdown",
"id": "95ce4240-f5a0-4810-8d09-b2f4c9e18b7f",
"metadata": {},
"source": [
"## Providing List of Valid Tools\n",
"\n",
"By default, the evaluator doesn't take into account the tools the agent is permitted to call. You can provide these to the evaluator via the `agent_tools` argument.\n"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "24c10566-2ef5-45c5-9213-a8fb28e2ca1f",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"from langchain.evaluation import load_evaluator\n",
"\n",
"evaluator = load_evaluator(\"trajectory\", agent_tools=[ping, trace_route])"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "7b995786-5b78-4d9e-8e8a-1f2a203113e2",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"{'score': 1.0,\n",
" 'reasoning': \"i. The final answer is helpful. It directly answers the user's question about the latency for the specified website.\\n\\nii. The AI language model uses a logical sequence of tools to answer the question. In this case, only one tool was needed to answer the question, and the model chose the correct one.\\n\\niii. The AI language model uses the tool in a helpful way. The 'ping' tool was used to determine the latency of the website, which was the information the user was seeking.\\n\\niv. The AI language model does not use too many steps to answer the question. Only one step was needed and used.\\n\\nv. The appropriate tool was used to answer the question. The 'ping' tool is designed to measure latency, which was the information the user was seeking.\\n\\nGiven these considerations, the AI language model's performance in answering this question is excellent.\"}"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"evaluation_result = evaluator.evaluate_agent_trajectory(\n",
" prediction=result[\"output\"],\n",
" input=result[\"input\"],\n",
" agent_trajectory=result[\"intermediate_steps\"],\n",
")\n",
"evaluation_result"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.2"
}
},
"nbformat": 4,
"nbformat_minor": 5
"cells": [
{
"cell_type": "markdown",
"id": "6e5ea1a1-7e74-459b-bf14-688f87d09124",
"metadata": {
"tags": []
},
"source": [
"# Agent Trajectory\n",
"[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain/blob/master/docs/docs/guides/evaluation/trajectory/trajectory_eval.ipynb)\n",
"\n",
"Agents can be difficult to holistically evaluate due to the breadth of actions and generation they can make. We recommend using multiple evaluation techniques appropriate to your use case. One way to evaluate an agent is to look at the whole trajectory of actions taken along with their responses.\n",
"\n",
"Evaluators that do this can implement the `AgentTrajectoryEvaluator` interface. This walkthrough will show how to use the `trajectory` evaluator to grade an OpenAI functions agent.\n",
"\n",
"For more information, check out the reference docs for the [TrajectoryEvalChain](https://api.python.langchain.com/en/latest/evaluation/langchain.evaluation.agents.trajectory_eval_chain.TrajectoryEvalChain.html#langchain.evaluation.agents.trajectory_eval_chain.TrajectoryEvalChain) for more info."
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "149402da-5212-43e2-b7c0-a701727f5293",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"from langchain.evaluation import load_evaluator\n",
"\n",
"evaluator = load_evaluator(\"trajectory\")"
]
},
{
"cell_type": "markdown",
"id": "b1c64c1a",
"metadata": {},
"source": [
"## Methods\n",
"\n",
"\n",
"The Agent Trajectory Evaluators are used with the [evaluate_agent_trajectory](https://api.python.langchain.com/en/latest/evaluation/langchain.evaluation.agents.trajectory_eval_chain.TrajectoryEvalChain.html#langchain.evaluation.agents.trajectory_eval_chain.TrajectoryEvalChain.evaluate_agent_trajectory) (and async [aevaluate_agent_trajectory](https://api.python.langchain.com/en/latest/evaluation/langchain.evaluation.agents.trajectory_eval_chain.TrajectoryEvalChain.html#langchain.evaluation.agents.trajectory_eval_chain.TrajectoryEvalChain.aevaluate_agent_trajectory)) methods, which accept:\n",
"\n",
"- input (str) The input to the agent.\n",
"- prediction (str) The final predicted response.\n",
"- agent_trajectory (List[Tuple[AgentAction, str]]) The intermediate steps forming the agent trajectory\n",
"\n",
"They return a dictionary with the following values:\n",
"- score: Float from 0 to 1, where 1 would mean \"most effective\" and 0 would mean \"least effective\"\n",
"- reasoning: String \"chain of thought reasoning\" from the LLM generated prior to creating the score"
]
},
{
"cell_type": "markdown",
"id": "e733562c-4c17-4942-9647-acfc5ebfaca2",
"metadata": {},
"source": [
"## Capturing Trajectory\n",
"\n",
"The easiest way to return an agent's trajectory (without using tracing callbacks like those in LangSmith) for evaluation is to initialize the agent with `return_intermediate_steps=True`.\n",
"\n",
"Below, create an example agent we will call to evaluate."
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "451cb0cb-6f42-4abd-aa6d-fb871fce034d",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import subprocess\n",
"\n",
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.tools import tool\n",
"from langchain.agents import AgentType, initialize_agent\n",
"\n",
"from pydantic import HttpUrl\n",
"from urllib.parse import urlparse\n",
"\n",
"\n",
"@tool\n",
"def ping(url: HttpUrl, return_error: bool) -> str:\n",
" \"\"\"Ping the fully specified url. Must include https:// in the url.\"\"\"\n",
" hostname = urlparse(str(url)).netloc\n",
" completed_process = subprocess.run(\n",
" [\"ping\", \"-c\", \"1\", hostname], capture_output=True, text=True\n",
" )\n",
" output = completed_process.stdout\n",
" if return_error and completed_process.returncode != 0:\n",
" return completed_process.stderr\n",
" return output\n",
"\n",
"\n",
"@tool\n",
"def trace_route(url: HttpUrl, return_error: bool) -> str:\n",
" \"\"\"Trace the route to the specified url. Must include https:// in the url.\"\"\"\n",
" hostname = urlparse(str(url)).netloc\n",
" completed_process = subprocess.run(\n",
" [\"traceroute\", hostname], capture_output=True, text=True\n",
" )\n",
" output = completed_process.stdout\n",
" if return_error and completed_process.returncode != 0:\n",
" return completed_process.stderr\n",
" return output\n",
"\n",
"\n",
"llm = ChatOpenAI(model=\"gpt-3.5-turbo-0613\", temperature=0)\n",
"agent = initialize_agent(\n",
" llm=llm,\n",
" tools=[ping, trace_route],\n",
" agent=AgentType.OPENAI_MULTI_FUNCTIONS,\n",
" return_intermediate_steps=True, # IMPORTANT!\n",
")\n",
"\n",
"result = agent(\"What's the latency like for https://langchain.com?\")"
]
},
{
"cell_type": "markdown",
"id": "2df34eed-45a5-4f91-88d3-9aa55f28391a",
"metadata": {
"tags": []
},
"source": [
"## Evaluate Trajectory\n",
"\n",
"Pass the input, trajectory, and pass to the [evaluate_agent_trajectory](https://api.python.langchain.com/en/latest/evaluation/langchain.evaluation.schema.AgentTrajectoryEvaluator.html#langchain.evaluation.schema.AgentTrajectoryEvaluator.evaluate_agent_trajectory) method."
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "8d2c8703-98ed-4068-8a8b-393f0f1f64ea",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"{'score': 1.0,\n",
" 'reasoning': \"i. The final answer is helpful. It directly answers the user's question about the latency for the website https://langchain.com.\\n\\nii. The AI language model uses a logical sequence of tools to answer the question. It uses the 'ping' tool to measure the latency of the website, which is the correct tool for this task.\\n\\niii. The AI language model uses the tool in a helpful way. It inputs the URL into the 'ping' tool and correctly interprets the output to provide the latency in milliseconds.\\n\\niv. The AI language model does not use too many steps to answer the question. It only uses one step, which is appropriate for this type of question.\\n\\nv. The appropriate tool is used to answer the question. The 'ping' tool is the correct tool to measure website latency.\\n\\nGiven these considerations, the AI language model's performance is excellent. It uses the correct tool, interprets the output correctly, and provides a helpful and direct answer to the user's question.\"}"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"evaluation_result = evaluator.evaluate_agent_trajectory(\n",
" prediction=result[\"output\"],\n",
" input=result[\"input\"],\n",
" agent_trajectory=result[\"intermediate_steps\"],\n",
")\n",
"evaluation_result"
]
},
{
"cell_type": "markdown",
"id": "fc5467c1-ea92-405f-949a-3011388fa9ee",
"metadata": {},
"source": [
"## Configuring the Evaluation LLM\n",
"\n",
"If you don't select an LLM to use for evaluation, the [load_evaluator](https://api.python.langchain.com/en/latest/evaluation/langchain.evaluation.loading.load_evaluator.html#langchain.evaluation.loading.load_evaluator) function will use `gpt-4` to power the evaluation chain. You can select any chat model for the agent trajectory evaluator as below."
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "1f6318f3-642a-4766-bc7a-f91239795ee7",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# %pip install anthropic\n",
"# ANTHROPIC_API_KEY=<YOUR ANTHROPIC API KEY>"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "b2852289-5df9-402e-95b5-7efebf0fc943",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"from langchain.chat_models import ChatAnthropic\n",
"\n",
"eval_llm = ChatAnthropic(temperature=0)\n",
"evaluator = load_evaluator(\"trajectory\", llm=eval_llm)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "ff72d21a-93b9-4c2f-8613-733d9c9330d7",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"{'score': 1.0,\n",
" 'reasoning': \"Here is my detailed evaluation of the AI's response:\\n\\ni. The final answer is helpful, as it directly provides the latency measurement for the requested website.\\n\\nii. The sequence of using the ping tool to measure latency is logical for this question.\\n\\niii. The ping tool is used in a helpful way, with the website URL provided as input and the output latency measurement extracted.\\n\\niv. Only one step is used, which is appropriate for simply measuring latency. More steps are not needed.\\n\\nv. The ping tool is an appropriate choice to measure latency. \\n\\nIn summary, the AI uses an optimal single step approach with the right tool and extracts the needed output. The final answer directly answers the question in a helpful way.\\n\\nOverall\"}"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"evaluation_result = evaluator.evaluate_agent_trajectory(\n",
" prediction=result[\"output\"],\n",
" input=result[\"input\"],\n",
" agent_trajectory=result[\"intermediate_steps\"],\n",
")\n",
"evaluation_result"
]
},
{
"cell_type": "markdown",
"id": "95ce4240-f5a0-4810-8d09-b2f4c9e18b7f",
"metadata": {},
"source": [
"## Providing List of Valid Tools\n",
"\n",
"By default, the evaluator doesn't take into account the tools the agent is permitted to call. You can provide these to the evaluator via the `agent_tools` argument.\n"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "24c10566-2ef5-45c5-9213-a8fb28e2ca1f",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"from langchain.evaluation import load_evaluator\n",
"\n",
"evaluator = load_evaluator(\"trajectory\", agent_tools=[ping, trace_route])"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "7b995786-5b78-4d9e-8e8a-1f2a203113e2",
"metadata": {
"tags": []
},
"outputs": [
{
"data": {
"text/plain": [
"{'score': 1.0,\n",
" 'reasoning': \"i. The final answer is helpful. It directly answers the user's question about the latency for the specified website.\\n\\nii. The AI language model uses a logical sequence of tools to answer the question. In this case, only one tool was needed to answer the question, and the model chose the correct one.\\n\\niii. The AI language model uses the tool in a helpful way. The 'ping' tool was used to determine the latency of the website, which was the information the user was seeking.\\n\\niv. The AI language model does not use too many steps to answer the question. Only one step was needed and used.\\n\\nv. The appropriate tool was used to answer the question. The 'ping' tool is designed to measure latency, which was the information the user was seeking.\\n\\nGiven these considerations, the AI language model's performance in answering this question is excellent.\"}"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"evaluation_result = evaluator.evaluate_agent_trajectory(\n",
" prediction=result[\"output\"],\n",
" input=result[\"input\"],\n",
" agent_trajectory=result[\"intermediate_steps\"],\n",
")\n",
"evaluation_result"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.2"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

@ -265,10 +265,16 @@
"cell_type": "code",
"execution_count": null,
"id": "5eba38dc",
"metadata": {},
"metadata": {
"vscode": {
"languageId": "plaintext"
}
},
"outputs": [],
"source": [
"CMAKE_ARGS=\"-DLLAMA_METAL=on\" FORCE_CMAKE=1 pip install -U llama-cpp-python --no-cache-dirclear"
"%env CMAKE_ARGS=\"-DLLAMA_METAL=on\"\n",
"%env FORCE_CMAKE=1\n",
"%pip install -U llama-cpp-python --no-cache-dirclear`"
]
},
{
@ -379,7 +385,7 @@
"metadata": {},
"outputs": [],
"source": [
"pip install gpt4all"
"pip install gpt4all\n"
]
},
{

@ -19,7 +19,8 @@
"metadata": {},
"outputs": [],
"source": [
"from langchain.chains import LLMChain\nfrom langchain.llms import OpenAI, Cohere, HuggingFaceHub\nfrom langchain.prompts import PromptTemplate\n",
"from langchain.llms import OpenAI, Cohere, HuggingFaceHub\n",
"from langchain.prompts import PromptTemplate\n",
"from langchain.model_laboratory import ModelLaboratory"
]
},
@ -139,7 +140,8 @@
"metadata": {},
"outputs": [],
"source": [
"from langchain.chains import SelfAskWithSearchChain\nfrom langchain.utilities import SerpAPIWrapper\n",
"from langchain.chains import SelfAskWithSearchChain\n",
"from langchain.utilities import SerpAPIWrapper\n",
"\n",
"open_ai_llm = OpenAI(temperature=0)\n",
"search = SerpAPIWrapper()\n",

@ -98,7 +98,7 @@
"metadata": {},
"outputs": [],
"source": [
"text = f\"\"\"Slim Shady recently lost his wallet. \n",
"text = \"\"\"Slim Shady recently lost his wallet. \n",
"Inside is some cash and his credit card with the number 4916 0387 9536 0861. \n",
"If you would find it, please call at 313-666-7440 or write an email here: real.slim.shady@gmail.com.\"\"\""
]

@ -78,7 +78,8 @@
"from langchain_experimental.comprehend_moderation import AmazonComprehendModerationChain\n",
"\n",
"comprehend_moderation = AmazonComprehendModerationChain(\n",
" client=comprehend_client, verbose=True # optional\n",
" client=comprehend_client,\n",
" verbose=True, # optional\n",
")"
]
},

@ -147,7 +147,7 @@
" api_key=os.environ[\"ARGILLA_API_KEY\"],\n",
")\n",
"\n",
"dataset.push_to_argilla(\"langchain-dataset\");"
"dataset.push_to_argilla(\"langchain-dataset\")"
]
},
{

@ -118,7 +118,6 @@
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"from langchain.callbacks.confident_callback import DeepEvalCallbackHandler\n",
"\n",
"deepeval_callback = DeepEvalCallbackHandler(\n",

@ -36,14 +36,10 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"vscode": {
"languageId": "shellscript"
}
},
"metadata": {},
"outputs": [],
"source": [
"$ pip install context-python --upgrade"
"!pip install context-python --upgrade"
]
},
{

@ -49,9 +49,7 @@
"from langchain.llms import OpenAI\n",
"import matplotlib.pyplot as plt\n",
"import matplotlib.dates as md\n",
"import os\n",
"import time\n",
"import sys\n",
"\n",
"from infinopy import InfinoClient\n",
"from langchain.callbacks import InfinoCallbackHandler"

@ -80,7 +80,6 @@
"from langchain.prompts import PromptTemplate\n",
"from langchain.chains import LLMChain, SimpleSequentialChain\n",
"from langchain.agents import initialize_agent, load_tools\n",
"from langchain.agents import Tool\n",
"from langchain.callbacks import SageMakerCallbackHandler\n",
"\n",
"from sagemaker.analytics import ExperimentAnalytics\n",

@ -266,7 +266,6 @@
"outputs": [],
"source": [
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.prompts import PromptTemplate\n",
"from langchain.schema import HumanMessage, SystemMessage\n",
"from langchain.callbacks import TrubricsCallbackHandler"
]

@ -20,13 +20,7 @@
"outputs": [],
"source": [
"from langchain.chat_models import ChatAnthropic\n",
"from langchain.prompts.chat import (\n",
" ChatPromptTemplate,\n",
" SystemMessagePromptTemplate,\n",
" AIMessagePromptTemplate,\n",
" HumanMessagePromptTemplate,\n",
")\n",
"from langchain.schema import AIMessage, HumanMessage, SystemMessage"
"from langchain.schema import HumanMessage"
]
},
{

@ -137,21 +137,13 @@
"source": [
"model0613 = AzureChatOpenAI(\n",
" openai_api_version=\"2023-05-15\",\n",
" deployment_name=\"gpt-35-turbo,\n",
" deployment_name=\"gpt-35-turbo\",\n",
" model_version=\"0613\",\n",
")\n",
"with get_openai_callback() as cb:\n",
" model0613([message])\n",
" print(f\"Total Cost (USD): ${format(cb.total_cost, '.6f')}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "99682534",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {

@ -226,7 +226,7 @@
"source": [
"res = chat.stream(\n",
" [HumanMessage(content=\"hi\")],\n",
" **{\"top_p\": 0.4, \"temperature\": 0.1, \"penalty_score\": 1}\n",
" **{\"top_p\": 0.4, \"temperature\": 0.1, \"penalty_score\": 1},\n",
")\n",
"\n",
"for r in res:\n",

@ -20,7 +20,7 @@
"outputs": [],
"source": [
"from langchain.chat_models import ChatCohere\n",
"from langchain.schema import AIMessage, HumanMessage"
"from langchain.schema import HumanMessage"
]
},
{

@ -23,10 +23,9 @@
"from langchain.prompts.chat import (\n",
" ChatPromptTemplate,\n",
" SystemMessagePromptTemplate,\n",
" AIMessagePromptTemplate,\n",
" HumanMessagePromptTemplate,\n",
")\n",
"from langchain.schema import AIMessage, HumanMessage, SystemMessage"
"from langchain.schema import HumanMessage, SystemMessage"
]
},
{
@ -139,11 +138,11 @@
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",

@ -34,13 +34,7 @@
"outputs": [],
"source": [
"from langchain.chat_models import ChatKonko\n",
"from langchain.prompts.chat import (\n",
" ChatPromptTemplate,\n",
" SystemMessagePromptTemplate,\n",
" AIMessagePromptTemplate,\n",
" HumanMessagePromptTemplate,\n",
")\n",
"from langchain.schema import AIMessage, HumanMessage, SystemMessage"
"from langchain.schema import HumanMessage, SystemMessage"
]
},
{

@ -23,13 +23,7 @@
"outputs": [],
"source": [
"from langchain.chat_models import ChatLiteLLM\n",
"from langchain.prompts.chat import (\n",
" ChatPromptTemplate,\n",
" SystemMessagePromptTemplate,\n",
" AIMessagePromptTemplate,\n",
" HumanMessagePromptTemplate,\n",
")\n",
"from langchain.schema import AIMessage, HumanMessage, SystemMessage"
"from langchain.schema import HumanMessage"
]
},
{

@ -23,10 +23,9 @@
"from langchain.prompts.chat import (\n",
" ChatPromptTemplate,\n",
" SystemMessagePromptTemplate,\n",
" AIMessagePromptTemplate,\n",
" HumanMessagePromptTemplate,\n",
")\n",
"from langchain.schema import AIMessage, HumanMessage, SystemMessage"
"from langchain.schema import HumanMessage, SystemMessage"
]
},
{

@ -119,7 +119,7 @@
}
],
"source": [
"from langchain.schema import AIMessage, HumanMessage, SystemMessage\n",
"from langchain.schema import HumanMessage, SystemMessage\n",
"\n",
"messages = [\n",
" SystemMessage(\n",

@ -25,10 +25,9 @@
"from langchain.prompts.chat import (\n",
" ChatPromptTemplate,\n",
" SystemMessagePromptTemplate,\n",
" AIMessagePromptTemplate,\n",
" HumanMessagePromptTemplate,\n",
")\n",
"from langchain.schema import AIMessage, HumanMessage, SystemMessage"
"from langchain.schema import HumanMessage, SystemMessage"
]
},
{

@ -36,16 +36,9 @@
"outputs": [],
"source": [
"import os.path\n",
"import base64\n",
"import json\n",
"import re\n",
"import time\n",
"from google.auth.transport.requests import Request\n",
"from google.oauth2.credentials import Credentials\n",
"from google_auth_oauthlib.flow import InstalledAppFlow\n",
"from googleapiclient.discovery import build\n",
"import logging\n",
"import requests\n",
"\n",
"SCOPES = [\"https://www.googleapis.com/auth/gmail.readonly\"]\n",
"\n",

@ -148,7 +148,6 @@
"metadata": {},
"outputs": [],
"source": [
"from langchain.schema import Document\n",
"from langchain.vectorstores import Chroma\n",
"from langchain.embeddings import OpenAIEmbeddings\n",
"from langchain.llms import OpenAI\n",

@ -118,7 +118,9 @@
"source": [
"loader = DocusaurusLoader(\n",
" \"https://python.langchain.com\",\n",
" filter_urls=[\"https://python.langchain.com/docs/integrations/document_loaders/sitemap\"],\n",
" filter_urls=[\n",
" \"https://python.langchain.com/docs/integrations/document_loaders/sitemap\"\n",
" ],\n",
")\n",
"documents = loader.load()"
]
@ -162,9 +164,11 @@
"source": [
"loader = DocusaurusLoader(\n",
" \"https://python.langchain.com\",\n",
" filter_urls=[\"https://python.langchain.com/docs/integrations/document_loaders/sitemap\"],\n",
" filter_urls=[\n",
" \"https://python.langchain.com/docs/integrations/document_loaders/sitemap\"\n",
" ],\n",
" # This will only include the content that matches these tags, otherwise they will be removed\n",
" custom_html_tags=[\"#content\", \".main\"]\n",
" custom_html_tags=[\"#content\", \".main\"],\n",
")"
]
},
@ -213,7 +217,9 @@
"source": [
"loader = DocusaurusLoader(\n",
" \"https://python.langchain.com\",\n",
" filter_urls=[\"https://python.langchain.com/docs/integrations/document_loaders/sitemap\"],\n",
" filter_urls=[\n",
" \"https://python.langchain.com/docs/integrations/document_loaders/sitemap\"\n",
" ],\n",
" parsing_function=remove_nav_and_header_elements,\n",
")"
]

@ -26,15 +26,11 @@
"\n",
"from langchain.document_loaders.figma import FigmaFileLoader\n",
"\n",
"from langchain.text_splitter import CharacterTextSplitter\n",
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.indexes import VectorstoreIndexCreator\n",
"from langchain.chains import ConversationChain, LLMChain\n",
"from langchain.memory import ConversationBufferWindowMemory\n",
"from langchain.prompts.chat import (\n",
" ChatPromptTemplate,\n",
" SystemMessagePromptTemplate,\n",
" AIMessagePromptTemplate,\n",
" HumanMessagePromptTemplate,\n",
")"
]

@ -61,7 +61,7 @@
"source": [
"loader = GoogleDriveLoader(\n",
" folder_id=\"1yucgL9WGgWZdM1TOuKkeghlPizuzMYb5\",\n",
" token_path='/path/where/you/want/token/to/be/created/google_token.json'\n",
" token_path=\"/path/where/you/want/token/to/be/created/google_token.json\",\n",
" # Optional: configure whether to recursively fetch files from subfolders. Defaults to False.\n",
" recursive=False,\n",
")"

@ -18,9 +18,6 @@
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"\n",
"from langchain.document_loaders import IuguLoader\n",
"from langchain.indexes import VectorstoreIndexCreator"
]

@ -21,9 +21,6 @@
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"\n",
"from langchain.document_loaders import ModernTreasuryLoader\n",
"from langchain.indexes import VectorstoreIndexCreator"
]

@ -1,66 +1,104 @@
{
"metadata": {
"vscode": {
"interpreter": {
"hash": "cc99336516f23363341912c6723b01ace86f02e26b4290be1efc0677e2e2ec24"
}
},
"kernelspec": {
"name": "python",
"display_name": "Python (Pyodide)",
"language": "python"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.1"
}
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Quip\n",
"\n",
">[Quip](https://quip.com) is a collaborative productivity software suite for mobile and Web. It allows groups of people to create and edit documents and spreadsheets as a group, typically for business purposes.\n",
"\n",
"A loader for `Quip` docs.\n",
"\n",
"Please refer [here](https://quip.com/dev/automation/documentation/current#section/Authentication/Get-Access-to-Quip's-APIs) to know how to get personal access token. \n",
"\n",
"Specify a list `folder_ids` and/or `thread_ids` to load in the corresponding docs into Document objects, if both are specified, loader will get all `thread_ids` belong to this folder based on `folder_ids`, combine with passed `thread_ids`, the union of both sets will be returned.\n",
"\n",
"* How to know folder_id ? \n",
" go to quip folder, right click folder and copy link, extract suffix from link as folder_id. Hint: `https://example.quip.com/<folder_id>`\n",
"* How to know thread_id ? \n",
" thread_id is the document id. Go to quip doc, right click doc and copy link, extract suffix from link as thread_id. Hint: `https://exmaple.quip.com/<thread_id>`\n",
" \n",
"You can also set `include_all_folders` as `True` will fetch group_folder_ids and \n",
"You can also specify a boolean `include_attachments` to include attachments, this is set to False by default, if set to True all attachments will be downloaded and QuipLoader will extract the text from the attachments and add it to the Document object. Currently supported attachment types are: `PDF`, `PNG`, `JPEG/JPG`, `SVG`, `Word` and `Excel`. Also you can sepcify a boolean `include_comments` to include comments in document, this is set to False by default, if set to True all comments in document will be fetched and QuipLoader will add them to Document objec.\n"
]
},
"nbformat_minor": 4,
"nbformat": 4,
"cells": [
{
"cell_type": "markdown",
"source": "# Quip\n\n>[Quip](https://quip.com) is a collaborative productivity software suite for mobile and Web. It allows groups of people to create and edit documents and spreadsheets as a group, typically for business purposes.\n\nA loader for `Quip` docs.\n\nPlease refer [here](https://quip.com/dev/automation/documentation/current#section/Authentication/Get-Access-to-Quip's-APIs) to know how to get personal access token. \n\nSpecify a list `folder_ids` and/or `thread_ids` to load in the corresponding docs into Document objects, if both are specified, loader will get all `thread_ids` belong to this folder based on `folder_ids`, combine with passed `thread_ids`, the union of both sets will be returned.\n\n* How to know folder_id ? \n go to quip folder, right click folder and copy link, extract suffix from link as folder_id. Hint: `https://example.quip.com/<folder_id>`\n* How to know thread_id ? \n thread_id is the document id. Go to quip doc, right click doc and copy link, extract suffix from link as thread_id. Hint: `https://exmaple.quip.com/<thread_id>`\n \nYou can also set `include_all_folders` as `True` will fetch group_folder_ids and \nYou can also specify a boolean `include_attachments` to include attachments, this is set to False by default, if set to True all attachments will be downloaded and QuipLoader will extract the text from the attachments and add it to the Document object. Currently supported attachment types are: `PDF`, `PNG`, `JPEG/JPG`, `SVG`, `Word` and `Excel`. Also you can sepcify a boolean `include_comments` to include comments in document, this is set to False by default, if set to True all comments in document will be fetched and QuipLoader will add them to Document objec.\n",
"metadata": {}
},
{
"cell_type": "markdown",
"source": "Before using QuipLoader make sure you have the latest version of the quip-api package installed:",
"metadata": {}
},
{
"cell_type": "code",
"source": "#!pip install quip-api",
"metadata": {
"tags": []
},
"execution_count": 1,
"outputs": []
},
{
"cell_type": "markdown",
"source": "## Examples",
"metadata": {}
},
{
"cell_type": "markdown",
"source": "### Personal Access Token",
"metadata": {}
},
{
"cell_type": "code",
"source": "from langchain.document_loaders import QuipLoader\n\nloader = QuipLoader(api_url=\"https://platform.quip.com\", access_token=\"change_me\", request_timeout=60)\ndocuments = loader.load(\n folder_ids={'123','456'}, thread_ids={'abc', 'efg'} include_attachments=False, include_comments=False\n)",
"metadata": {},
"execution_count": null,
"outputs": []
}
]
}
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Before using QuipLoader make sure you have the latest version of the quip-api package installed:"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"#!pip install quip-api\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Examples"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Personal Access Token"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from langchain.document_loaders import QuipLoader\n",
"\n",
"loader = QuipLoader(\n",
" api_url=\"https://platform.quip.com\", access_token=\"change_me\", request_timeout=60\n",
")\n",
"documents = loader.load(\n",
" folder_ids={\"123\", \"456\"},\n",
" thread_ids={\"abc\", \"efg\"},\n",
" include_attachments=False,\n",
" include_comments=False,\n",
")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python (Pyodide)",
"language": "python",
"name": "python"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.1"
},
"vscode": {
"interpreter": {
"hash": "cc99336516f23363341912c6723b01ace86f02e26b4290be1efc0677e2e2ec24"
}
}
},
"nbformat": 4,
"nbformat_minor": 4
}

@ -29,14 +29,10 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"vscode": {
"languageId": "shellscript"
}
},
"metadata": {},
"outputs": [],
"source": [
"$ pip3 install rockset"
"!pip install rockset"
]
},
{

@ -18,9 +18,6 @@
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"\n",
"from langchain.document_loaders import StripeLoader\n",
"from langchain.indexes import VectorstoreIndexCreator"
]

@ -91,7 +91,7 @@
"# Example: transcript = soup.select_one(\"td[class='scrtext']\").text\n",
"# BS4 documentation can be found here: https://www.crummy.com/software/BeautifulSoup/bs4/doc/\n",
"\n",
"\"\"\";"
"\"\"\""
]
},
{
@ -226,19 +226,23 @@
},
{
"cell_type": "markdown",
"id": "672264ad",
"metadata": {
"collapsed": false
},
"source": [
"## Using proxies\n",
"\n",
"Sometimes you might need to use proxies to get around IP blocks. You can pass in a dictionary of proxies to the loader (and `requests` underneath) to use them."
],
"metadata": {
"collapsed": false
},
"id": "672264ad"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9caf0310",
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"loader = WebBaseLoader(\n",
@ -249,11 +253,7 @@
" },\n",
")\n",
"docs = loader.load()"
],
"metadata": {
"collapsed": false
},
"id": "9caf0310"
]
}
],
"metadata": {

@ -109,7 +109,7 @@
"\n",
"print(\n",
" *[d.page_content + \"\\n\\n\" + json.dumps(d.metadata) for d in enhanced_documents],\n",
" sep=\"\\n\\n---------------\\n\\n\"\n",
" sep=\"\\n\\n---------------\\n\\n\",\n",
")"
]
},
@ -167,7 +167,7 @@
"\n",
"print(\n",
" *[d.page_content + \"\\n\\n\" + json.dumps(d.metadata) for d in enhanced_documents],\n",
" sep=\"\\n\\n---------------\\n\\n\"\n",
" sep=\"\\n\\n---------------\\n\\n\",\n",
")"
]
},
@ -225,7 +225,7 @@
"\n",
"print(\n",
" *[d.page_content + \"\\n\\n\" + json.dumps(d.metadata) for d in enhanced_documents],\n",
" sep=\"\\n\\n---------------\\n\\n\"\n",
" sep=\"\\n\\n---------------\\n\\n\",\n",
")"
]
},

@ -201,7 +201,6 @@
],
"source": [
"from langchain.llms.loading import load_llm\n",
"from langchain.llms.azureml_endpoint import AzureMLEndpointClient\n",
"\n",
"save_llm = AzureMLOnlineEndpoint(\n",
" deployment_name=\"databricks-dolly-v2-12b-4\",\n",

@ -227,7 +227,7 @@
"res = llm.generate(\n",
" prompts=[\"hi\"],\n",
" streaming=True,\n",
" **{\"top_p\": 0.4, \"temperature\": 0.1, \"penalty_score\": 1}\n",
" **{\"top_p\": 0.4, \"temperature\": 0.1, \"penalty_score\": 1},\n",
")\n",
"\n",
"for r in res:\n",

@ -37,7 +37,6 @@
"# * the model's url slug\n",
"\n",
"import os\n",
"from getpass import getpass\n",
"\n",
"# You can get this from the main dashboard\n",
"# at https://app.banana.dev\n",
@ -53,7 +52,8 @@
"outputs": [],
"source": [
"from langchain.llms import Banana\n",
"from langchain.prompts import PromptTemplate\nfrom langchain.chains import LLMChain"
"from langchain.prompts import PromptTemplate\n",
"from langchain.chains import LLMChain"
]
},
{

@ -2,6 +2,7 @@
"cells": [
{
"cell_type": "markdown",
"id": "34803e5e",
"metadata": {
"id": "J-yvaDTmTTza"
},
@ -11,51 +12,50 @@
"Calls the Beam API wrapper to deploy and make subsequent calls to an instance of the gpt2 LLM in a cloud deployment. Requires installation of the Beam library and registration of Beam Client ID and Client Secret. By calling the wrapper an instance of the model is created and run, with returned text relating to the prompt. Additional calls can then be made by directly calling the Beam API.\n",
"\n",
"[Create an account](https://www.beam.cloud/), if you don't have one already. Grab your API keys from the [dashboard](https://www.beam.cloud/dashboard/settings/api-keys)."
],
"id": "34803e5e"
]
},
{
"cell_type": "markdown",
"id": "76af7763",
"metadata": {
"id": "CfTmesWtTfTS"
},
"source": [
"Install the Beam CLI"
],
"id": "76af7763"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ef012b8d",
"metadata": {
"id": "G_tCCurqR7Ik"
},
"outputs": [],
"source": [
"!curl https://raw.githubusercontent.com/slai-labs/get-beam/main/get-beam.sh -sSfL | sh"
],
"id": "ef012b8d"
]
},
{
"cell_type": "markdown",
"id": "74be8c2e",
"metadata": {
"id": "jJkcNqOdThQ7"
},
"source": [
"Register API Keys and set your beam client id and secret environment variables:"
],
"id": "74be8c2e"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2a176107",
"metadata": {
"id": "7gQd6fszSEaH"
},
"outputs": [],
"source": [
"import os\n",
"import subprocess\n",
"\n",
"beam_client_id = \"<Your beam client id>\"\n",
"beam_client_secret = \"<Your beam client secret>\"\n",
@ -66,33 +66,33 @@
"\n",
"# Run the beam configure command\n",
"!beam configure --clientId={beam_client_id} --clientSecret={beam_client_secret}"
],
"id": "2a176107"
]
},
{
"cell_type": "markdown",
"id": "64cc18b3",
"metadata": {
"id": "c20rkK18TrK2"
},
"source": [
"Install the Beam SDK:"
],
"id": "64cc18b3"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a0014676",
"metadata": {
"id": "CH2Vop6ISNIf"
},
"outputs": [],
"source": [
"!pip install beam-sdk"
],
"id": "a0014676"
]
},
{
"cell_type": "markdown",
"id": "a48d515c",
"metadata": {
"id": "XflOsp3bTwl1"
},
@ -100,12 +100,12 @@
"**Deploy and call Beam directly from langchain!**\n",
"\n",
"Note that a cold start might take a couple of minutes to return the response, but subsequent calls will be faster!"
],
"id": "a48d515c"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c79e740b",
"metadata": {
"id": "KmaHxUqbSVnh"
},
@ -138,8 +138,7 @@
"response = llm._call(\"Running machine learning on a remote GPU\")\n",
"\n",
"print(response)"
],
"id": "c79e740b"
]
}
],
"metadata": {

@ -120,17 +120,12 @@
"outputs": [],
"source": [
"from langchain.agents import (\n",
" AgentType,\n",
" initialize_agent,\n",
" load_tools,\n",
" ZeroShotAgent,\n",
" Tool,\n",
" AgentExecutor,\n",
")\n",
"from langchain.memory import ConversationBufferMemory\n",
"from langchain.chains import LLMChain\n",
"from langchain.prompts import PromptTemplate\n",
"from langchain.utilities import GoogleSearchAPIWrapper, SerpAPIWrapper\n",
"from langchain.llms import NIBittensorLLM\n",
"\n",
"memory = ConversationBufferMemory(memory_key=\"chat_history\")\n",

@ -22,7 +22,8 @@
"outputs": [],
"source": [
"from langchain.llms import ChatGLM\n",
"from langchain.prompts import PromptTemplate\nfrom langchain.chains import LLMChain\n",
"from langchain.prompts import PromptTemplate\n",
"from langchain.chains import LLMChain\n",
"\n",
"# import os"
]
@ -52,7 +53,9 @@
"llm = ChatGLM(\n",
" endpoint_url=endpoint_url,\n",
" max_token=80000,\n",
" history=[[\"我将从美国到中国来旅游,出行前希望了解中国的城市\", \"欢迎问我任何问题。\"]],\n",
" history=[\n",
" [\"我将从美国到中国来旅游,出行前希望了解中国的城市\", \"欢迎问我任何问题。\"]\n",
" ],\n",
" top_p=0.9,\n",
" model_kwargs={\"sample_model_args\": False},\n",
")\n",

@ -32,16 +32,10 @@
"which you can get by creating an account https://app.edenai.run/user/register and heading here https://app.edenai.run/admin/account/settings\n",
"\n",
"Once we have a key we'll want to set it as an environment variable by running:\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"export EDENAI_API_KEY=\"...\""
"\n",
"```bash\n",
"export EDENAI_API_KEY=\"...\"\n",
"```"
]
},
{
@ -138,7 +132,6 @@
"import base64\n",
"from io import BytesIO\n",
"from PIL import Image\n",
"import json\n",
"\n",
"\n",
"def print_base64_image(base64_string):\n",

@ -21,11 +21,6 @@
"source": [
"from langchain.llms.fireworks import Fireworks\n",
"from langchain.prompts import PromptTemplate\n",
"from langchain.chains import LLMChain\n",
"from langchain.prompts.chat import (\n",
" ChatPromptTemplate,\n",
" HumanMessagePromptTemplate,\n",
")\n",
"import os"
]
},

@ -16,7 +16,7 @@
"metadata": {},
"source": [
"## Install openai\n",
"The `openai` package is required to use the GooseAI API. Install `openai` using `pip3 install openai`."
"The `openai` package is required to use the GooseAI API. Install `openai` using `pip install openai`."
]
},
{
@ -25,7 +25,7 @@
"metadata": {},
"outputs": [],
"source": [
"$ pip3 install openai"
"!pip install openai"
]
},
{
@ -43,7 +43,8 @@
"source": [
"import os\n",
"from langchain.llms import GooseAI\n",
"from langchain.prompts import PromptTemplate\nfrom langchain.chains import LLMChain"
"from langchain.prompts import PromptTemplate\n",
"from langchain.chains import LLMChain"
]
},
{

@ -141,7 +141,6 @@
],
"source": [
"from langchain.embeddings import JavelinAIGatewayEmbeddings\n",
"from langchain.embeddings.openai import OpenAIEmbeddings\n",
"\n",
"embeddings = JavelinAIGatewayEmbeddings(\n",
" gateway_uri=\"http://localhost:8000\", # replace with service URL or host/port of Javelin\n",

@ -59,7 +59,6 @@
},
"outputs": [],
"source": [
"from typing import Optional\n",
"from langchain.tools import tool\n",
"import os\n",
"import json\n",
@ -124,9 +123,7 @@
"BEGIN! Answer the Human's question as best as you are able.\n",
"------\n",
"Human: 'What's the difference between an iterator and an iterable?'\n",
"AI Assistant:\"\"\".format(\n",
" arg_schema=ask_star_coder.args\n",
")"
"AI Assistant:\"\"\".format(arg_schema=ask_star_coder.args)"
]
},
{

@ -1248,7 +1248,6 @@
"outputs": [],
"source": [
"from langchain.text_splitter import CharacterTextSplitter\n",
"from langchain.chains.mapreduce import MapReduceChain\n",
"\n",
"text_splitter = CharacterTextSplitter()"
]

@ -41,7 +41,6 @@
"source": [
"from getpass import getpass\n",
"import os\n",
"import subprocess\n",
"\n",
"\n",
"# Check if OPENAI_API_KEY environment variable is set\n",
@ -71,7 +70,8 @@
"outputs": [],
"source": [
"from langchain.llms import OpenLM\n",
"from langchain.prompts import PromptTemplate\nfrom langchain.chains import LLMChain"
"from langchain.prompts import PromptTemplate\n",
"from langchain.chains import LLMChain"
]
},
{
@ -106,9 +106,7 @@
" result = llm_chain.run(question)\n",
" print(\n",
" \"\"\"Model: {}\n",
"Result: {}\"\"\".format(\n",
" model, result\n",
" )\n",
"Result: {}\"\"\".format(model, result)\n",
" )"
]
}

@ -2,27 +2,28 @@
"cells": [
{
"cell_type": "markdown",
"id": "3f0a201c",
"metadata": {},
"source": [
"# Prediction Guard"
],
"id": "3f0a201c"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4f810331",
"metadata": {
"id": "3RqWPav7AtKL"
},
"outputs": [],
"source": [
"! pip install predictionguard langchain"
],
"id": "4f810331"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7191a5ce",
"metadata": {
"id": "2xe8JEUwA7_y"
},
@ -30,26 +31,26 @@
"source": [
"import os\n",
"\n",
"import predictionguard as pg\n",
"from langchain.llms import PredictionGuard\n",
"from langchain.prompts import PromptTemplate\nfrom langchain.chains import LLMChain"
],
"id": "7191a5ce"
"from langchain.prompts import PromptTemplate\n",
"from langchain.chains import LLMChain"
]
},
{
"cell_type": "markdown",
"id": "a8d356d3",
"metadata": {
"id": "mesCTyhnJkNS"
},
"source": [
"## Basic LLM usage\n",
"\n"
],
"id": "a8d356d3"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "158b109a",
"metadata": {
"id": "kp_Ymnx1SnDG"
},
@ -61,46 +62,46 @@
"\n",
"# Your Prediction Guard API key. Get one at predictionguard.com\n",
"os.environ[\"PREDICTIONGUARD_TOKEN\"] = \"<your Prediction Guard access token>\""
],
"id": "158b109a"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "140717c9",
"metadata": {
"id": "Ua7Mw1N4HcER"
},
"outputs": [],
"source": [
"pgllm = PredictionGuard(model=\"OpenAI-text-davinci-003\")"
],
"id": "140717c9"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "605f7ab6",
"metadata": {
"id": "Qo2p5flLHxrB"
},
"outputs": [],
"source": [
"pgllm(\"Tell me a joke\")"
],
"id": "605f7ab6"
]
},
{
"cell_type": "markdown",
"id": "99de09f9",
"metadata": {
"id": "EyBYaP_xTMXH"
},
"source": [
"## Control the output structure/ type of LLMs"
],
"id": "99de09f9"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ae6bd8a1",
"metadata": {
"id": "55uxzhQSTPqF"
},
@ -118,12 +119,12 @@
"\n",
"Result: \"\"\"\n",
"prompt = PromptTemplate(template=template, input_variables=[\"query\"])"
],
"id": "ae6bd8a1"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f81be0fb",
"metadata": {
"id": "yersskWbTaxU"
},
@ -131,12 +132,12 @@
"source": [
"# Without \"guarding\" or controlling the output of the LLM.\n",
"pgllm(prompt.format(query=\"What kind of post is this?\"))"
],
"id": "f81be0fb"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0cb3b91f",
"metadata": {
"id": "PzxSbYwqTm2w"
},
@ -154,34 +155,34 @@
" },\n",
")\n",
"pgllm(prompt.format(query=\"What kind of post is this?\"))"
],
"id": "0cb3b91f"
]
},
{
"cell_type": "markdown",
"id": "c3b6211f",
"metadata": {
"id": "v3MzIUItJ8kV"
},
"source": [
"## Chaining"
],
"id": "c3b6211f"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8d57d1b5",
"metadata": {
"id": "pPegEZExILrT"
},
"outputs": [],
"source": [
"pgllm = PredictionGuard(model=\"OpenAI-text-davinci-003\")"
],
"id": "8d57d1b5"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7915b7fa",
"metadata": {
"id": "suxw62y-J-bg"
},
@ -196,12 +197,12 @@
"question = \"What NFL team won the Super Bowl in the year Justin Beiber was born?\"\n",
"\n",
"llm_chain.predict(question=question)"
],
"id": "7915b7fa"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "32ffd783",
"metadata": {
"id": "l2bc26KHKr7n"
},
@ -212,18 +213,17 @@
"llm_chain = LLMChain(prompt=prompt, llm=pgllm, verbose=True)\n",
"\n",
"llm_chain.predict(adjective=\"sad\", subject=\"ducks\")"
],
"id": "32ffd783"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "408ad1e1",
"metadata": {
"id": "I--eSa2PLGqq"
},
"outputs": [],
"source": [],
"id": "408ad1e1"
"source": []
}
],
"metadata": {

@ -34,7 +34,7 @@
},
"outputs": [],
"source": [
"pip install titan-iris"
"!pip install titan-iris"
]
},
{
@ -49,22 +49,13 @@
"## Taking off\n",
"Models are referred to by their model id on HuggingFace. Takeoff uses port 8000 by default, but can be configured to use another port. There is also support to use a Nvidia GPU by specifying cuda for the device flag.\n",
"\n",
"To start the takeoff server, run:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"vscode": {
"languageId": "shellscript"
}
},
"outputs": [],
"source": [
"To start the takeoff server, run:\n",
"\n",
"```shell\n",
"iris takeoff --model tiiuae/falcon-7b-instruct --device cpu\n",
"iris takeoff --model tiiuae/falcon-7b-instruct --device cuda # Nvidia GPU required\n",
"iris takeoff --model tiiuae/falcon-7b-instruct --device cpu --port 5000 # run on port 5000 (default: 8000)"
"iris takeoff --model tiiuae/falcon-7b-instruct --device cpu --port 5000 # run on port 5000 (default: 8000)\n",
"```"
]
},
{
@ -74,20 +65,11 @@
"You will then be directed to a login page, where you will need to create an account to proceed.\n",
"After logging in, run the command onscreen to check whether the server is ready. When it is ready, you can start using the Takeoff integration.\n",
"\n",
"To shutdown the server, run the following command. You will be presented with options on which Takeoff server to shut down, in case you have multiple running servers.\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"vscode": {
"languageId": "shellscript"
}
},
"outputs": [],
"source": [
"iris takeoff --shutdown # shutdown the server"
"To shutdown the server, run the following command. You will be presented with options on which Takeoff server to shut down, in case you have multiple running servers.\n",
"\n",
"```shell\n",
"iris takeoff --shutdown # shutdown the server\n",
"```"
]
},
{

@ -252,8 +252,7 @@
"from langchain.chat_models import ChatOpenAI\n",
"from langchain.agents import initialize_agent\n",
"from langchain.agents import AgentType\n",
"from langchain_experimental.utilities import PythonREPL\n",
"from getpass import getpass"
"from langchain_experimental.utilities import PythonREPL"
]
},
{

@ -50,7 +50,6 @@
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import getpass\n",
"\n",
"database_mode = (input(\"\\n(C)assandra or (A)stra DB? \")).upper()\n",

@ -25,7 +25,7 @@
" url=\"bolt://localhost:7687\",\n",
" username=\"neo4j\",\n",
" password=\"password\",\n",
" session_id=\"session_id_1\"\n",
" session_id=\"session_id_1\",\n",
")\n",
"\n",
"history.add_user_message(\"hi!\")\n",

@ -58,7 +58,8 @@
"history = RocksetChatMessageHistory(\n",
" session_id=\"MySession\",\n",
" client=RocksetClient(\n",
" api_key=\"YOUR API KEY\", host=Regions.usw2a1 # us-west-2 Oregon\n",
" api_key=\"YOUR API KEY\",\n",
" host=Regions.usw2a1, # us-west-2 Oregon\n",
" ),\n",
" collection=\"langchain_demo\",\n",
" sync=True,\n",

@ -61,23 +61,23 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": null,
"metadata": {
"id": "9Hq9snQasynA"
},
"outputs": [],
"source": [
"def make_langchain_chat_llm(chat_model=):\n",
"def make_langchain_chat_llm():\n",
" return ChatOpenAI(\n",
" streaming=True,\n",
" temperature=0.1,\n",
" callbacks=[\n",
" StreamingStdOutCallbackHandler(),\n",
" ArthurCallbackHandler.from_credentials(\n",
" arthur_model_id, \n",
" arthur_url=arthur_url, \n",
" arthur_login=arthur_login)\n",
" ])"
" arthur_model_id, arthur_url=arthur_url, arthur_login=arthur_login\n",
" ),\n",
" ],\n",
" )"
]
},
{

@ -105,7 +105,6 @@
}
],
"source": [
"from datetime import datetime\n",
"from langchain.callbacks import StdOutCallbackHandler\n",
"from langchain.llms import OpenAI\n",
"\n",

@ -49,7 +49,6 @@
"source": [
"%pip install comet_ml langchain openai google-search-results spacy textstat pandas\n",
"\n",
"import sys\n",
"\n",
"!{sys.executable} -m spacy download en_core_web_sm"
]
@ -119,8 +118,6 @@
"metadata": {},
"outputs": [],
"source": [
"from datetime import datetime\n",
"\n",
"from langchain.callbacks import CometCallbackHandler, StdOutCallbackHandler\n",
"from langchain.llms import OpenAI\n",
"\n",

@ -106,7 +106,6 @@
"from langchain.chains import LLMChain\n",
"from langchain.llms import OpenAI\n",
"from langchain.prompts import PromptTemplate\n",
"import os\n",
"\n",
"# Set up the LangChain SDK with the environment variable\n",
"llm = OpenAI(temperature=0)"

@ -21,7 +21,6 @@
"source": [
"import os\n",
"from langchain.vectorstores import Vectara\n",
"from langchain.vectorstores.vectara import VectaraRetriever\n",
"from langchain.llms import OpenAI\n",
"from langchain.chains import ConversationalRetrievalChain"
]

@ -27,7 +27,6 @@
"import os\n",
"from langchain.llms import OpenAI\n",
"from langchain.docstore.document import Document\n",
"import requests\n",
"from langchain.vectorstores import Vectara\n",
"from langchain.text_splitter import CharacterTextSplitter\n",
"from langchain.prompts import PromptTemplate\n",

@ -272,10 +272,9 @@
"metadata": {},
"outputs": [],
"source": [
"from typing import Optional, List\n",
"from typing import List\n",
"\n",
"from langchain.chains.openai_functions import (\n",
" create_openai_fn_chain,\n",
" create_structured_output_chain,\n",
")\n",
"from langchain.chat_models import ChatOpenAI\n",

@ -35,7 +35,6 @@
"metadata": {},
"outputs": [],
"source": [
"import boto3\n",
"from langchain.retrievers import AmazonKendraRetriever"
]
},

@ -33,7 +33,6 @@
"from langchain.retrievers import DocArrayRetriever\n",
"from docarray import BaseDoc\n",
"from docarray.typing import NdArray\n",
"import numpy as np\n",
"from langchain.embeddings import FakeEmbeddings\n",
"import random\n",
"\n",

@ -72,7 +72,6 @@
"source": [
"import os\n",
"from langchain.retrievers import KayAiRetriever\n",
"from kay.rag.retrievers import KayRetriever\n",
"\n",
"os.environ[\"KAY_API_KEY\"] = KAY_API_KEY\n",
"retriever = KayAiRetriever.create(\n",

@ -37,7 +37,7 @@
"CLIENT_ID = \"\"\n",
"INDEX_ID = \"\"\n",
"\n",
"metal = Metal(API_KEY, CLIENT_ID, INDEX_ID);"
"metal = Metal(API_KEY, CLIENT_ID, INDEX_ID)"
]
},
{

Some files were not shown because too many files have changed in this diff Show More

Loading…
Cancel
Save