# /// script # requires-python = ">=3.12,<3.15" # dependencies = [ # "marimo", # "requests", # "yfinance", # ] # /// """Python Handbook, Chapter D: Agents and APIs. """ import marimo __generated_with = "0.24.0" app = marimo.App(width="medium", sql_output="polars") @app.cell def _(): import json import random import re import marimo as mo import requests import yfinance as yf return json, mo, random, re, requests, yf @app.cell(hide_code=True) def _(mo): mo.md(r""" # Agents and APIs Chapter D of the Python Handbook. Nine sections, each one concept. Read this chapter and you can send a request to a model, read what comes back, give it a tool, run the loop, and calculate what the whole thing cost. Everything in this chapter is code you already know. A request is a dictionary. A reply is a dictionary. The conversation is a list of dictionaries. The loop is a `while`, covered in [section 12](https://oim6301.github.io/handbooks/python/02-collections/#12-iterating-over-a-list-with-for). There is no new syntax here, and the difficulty is entirely in knowing which dictionary holds what. **What you need to run the model code.** A key, which Babson issues and [Your AI Tools and Babson Key](https://oim6301.github.io/guides/babson-ai-key/) explains, and the Anthropic library, which is one command: ``` uv add anthropic ``` **Which code on this page runs, and which is quoted.** No cell here calls a model, because a published page cannot hold a key. Every model call sits in a grey block for you to copy. What the model would send back is on this page as the real JSON the API returns, built in a real cell, and every cell that reads it, dispatches it, loops over it or prices it is live. Section 31 and section 39 run end to end with nothing installed but this notebook. To run the code yourself, download the `.py` into your project and open it with `uv run marimo edit`. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## 31. HTTP requests An API is a web address that answers with data instead of a page. You send a request, a server sends back a response, and your job is to read the response before you assume anything about it. Every model call later in this chapter is one of these underneath. The first requests below go to Kraken, a cryptocurrency exchange whose public price data needs no key. Each one is wrapped in a `try` so the page still renders when there is no network, in which case it falls back to a body recorded on 2026-09-15 and says so. ๐Ÿ“– **Used in:** Getting data from an API """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Requests and responses `requests.get(url)` sends the request and hands you a response object. Three things on it matter: the **status code**, the **headers**, and the **body**. `timeout=10` says how long to wait before giving up. Leave it out and a server that never answers hangs your notebook until you kill it. """) return @app.cell def _(requests): TICKER_URL = "https://api.kraken.com/0/public/Ticker?pair=XBTUSD" TICKER_SAVED = '{"error":[],"result":{"XXBTZUSD":{"a":["76386.40000","1","1.000"],"b":["76386.30000","3","3.000"],"c":["76386.40000","0.00049007"],"v":["2309.89255277","3625.12550720"],"p":["76741.04546","77539.45124"],"t":[78754,118300],"l":["75561.00000","75561.00000"],"h":["78242.10000","79581.20000"],"o":"78184.20000"}}}' try: ticker_reply = requests.get(TICKER_URL, timeout=10) ticker_status = ticker_reply.status_code ticker_type = ticker_reply.headers["content-type"] ticker_body = ticker_reply.text ticker_source = "live request" except requests.RequestException: ticker_status = 200 ticker_type = "application/json" ticker_body = TICKER_SAVED ticker_source = "saved copy, no network" f"{ticker_source} | status {ticker_status} | {ticker_type} | {len(ticker_body)} characters" return (ticker_body,) @app.cell(hide_code=True) def _(mo): mo.md(r""" **200** means the server understood and answered. The content type says what the body is. The body itself is text, however much it looks like a Python dictionary. ๐Ÿ‘‰ **Try it:** print `ticker_body[:80]`. Then check `type(ticker_body)` and explain why the next knowledge point exists. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### `.json()` turns the body into Python The body arrives as one long string. `json.loads()` parses it into dictionaries and lists. The `requests` library wraps this as `response.json()`, which is the form you will see everywhere. """) return @app.cell def _(json, ticker_body): ticker = json.loads(ticker_body) type(ticker), list(ticker.keys()) return (ticker,) @app.cell(hide_code=True) def _(mo, ticker): mo.md(f""" The reply carries **{len(ticker)}** keys, and Kraken sends `error` on every answer; `result` comes only when the request worked. `error` is a list, and here it holds **{len(ticker['error'])}** items. `result` holds the prices. A reply is good only when `error` is empty and `result` is there. ๐Ÿ‘‰ **Try it:** put `ticker["result"]` in a cell of its own and read what comes back before you go any deeper. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Inspect before you index You asked for `XBTUSD`. Look at the name the answer is filed under. """) return @app.cell def _(ticker): result_keys = list(ticker["result"].keys()) result_keys return (result_keys,) @app.cell(hide_code=True) def _(mo, result_keys): mo.md(f""" The answer is filed under **{result_keys[0]}**. You sent `XBTUSD`, and Kraken renamed the pair into its own internal code on the way back. **The trap.** `ticker["result"]["XBTUSD"]` raises a `KeyError`, although the request worked and `XBTUSD` is exactly what you typed. Nothing in your code tells you what the key is called. The response does, and `list(reply.keys())` costs one line. ๐Ÿ‘‰ **Try it:** change the request to `pair=XBTUSD,ETHUSD` and predict both keys of `result` before you list them. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### A status code of 200 is not an answer Misspell the pair and send the request again. """) return @app.cell def _(requests): WRONG_PAIR_URL = "https://api.kraken.com/0/public/Ticker?pair=NOPE" try: wrong_pair_reply = requests.get(WRONG_PAIR_URL, timeout=10) wrong_pair_status = wrong_pair_reply.status_code wrong_pair_body = wrong_pair_reply.text wrong_pair_source = "live request" except requests.RequestException: wrong_pair_status = 200 wrong_pair_body = '{"error":["EQuery:Unknown asset pair"]}' wrong_pair_source = "saved copy, no network" wrong_pair_source, wrong_pair_status, wrong_pair_body return wrong_pair_body, wrong_pair_status @app.cell def _(json, wrong_pair_body): wrong_pair = json.loads(wrong_pair_body) list(wrong_pair.keys()), wrong_pair["error"] return (wrong_pair,) @app.cell def _(requests): WRONG_PATH_URL = "https://api.kraken.com/0/public/Tickers?pair=XBTUSD" try: wrong_path_status = requests.get(WRONG_PATH_URL, timeout=10).status_code wrong_path_source = "live request" except requests.RequestException: wrong_path_status = 404 wrong_path_source = "saved copy, no network" wrong_path_source, wrong_path_status return (wrong_path_status,) @app.cell(hide_code=True) def _(mo, wrong_pair, wrong_pair_status, wrong_path_status): mo.md(f""" The status is **{wrong_pair_status}**. The body parses cleanly into a dictionary with **{len(wrong_pair)}** key, `error`, and the message inside it is `{wrong_pair['error'][0]}`. There is no `result`, so the next line falls over: ```python wrong_pair["result"] ``` ``` Traceback (most recent call last): File "ticker.py", line 4, in wrong_pair["result"] ~~~~~~~~~~^^^^^^^^^^ KeyError: 'result' ``` Misspell the method instead, `Tickers` for `Ticker`, and the status is **{wrong_path_status}**. This server uses the status code for a wrong address and the `error` list for a wrong question, and code that checks only one of them misses the other. **The trap.** A `KeyError` is not a parsing failure. The JSON parsed. What was wrong was the assumption about what was in it. Last year's course materials caught this exception and printed *JSON parsing error*, which sent everybody who read it looking in the wrong place. Check the status code, then check `error`, and only then index. ๐Ÿ‘‰ **Try it:** write the lines that check `status_code == 200` and an empty `error` before reading `result`, and print something useful when either check fails. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### A price can arrive as text Look at what a price is here, and at what a count of trades is. """) return @app.cell def _(result_keys, ticker): last_trade = ticker["result"][result_keys[0]]["c"] trade_counts = ticker["result"][result_keys[0]]["t"] last_trade, trade_counts return last_trade, trade_counts @app.cell(hide_code=True) def _(last_trade, mo, trade_counts): mo.md(f""" `c` is Kraken's name for the last trade, and it holds **{len(last_trade)}** values: the price, then the volume. The price is `{last_trade[0]}`, and its type is `{type(last_trade[0]).__name__}`. In the same reply `t`, the number of trades today and over the last 24 hours, holds `{trade_counts}`, and those are `{type(trade_counts[0]).__name__}`. The server picked a type for each field, and it did not pick the same one. Chapter A section 2 is where this bites. `last_trade[0] * 2` repeats the text instead of doubling the price, and `last_trade[0] + 1` raises a `TypeError`. Check the type of a value before you do arithmetic with it, whatever it looks like. ๐Ÿ‘‰ **Try it:** convert the price to a float and work out what 0.1 bitcoin cost at that price. Then find out what `a` and `b` hold in the same reply. ### In Excel terms Power Query's **Get Data from Web** opens a JSON response, flattens it into columns and guesses a type for each one. Python does none of that. You get nested dictionaries and lists in the shape the server chose, and every value keeps the type the server sent, which for a lot of APIs means prices as text. The Excel habit that misleads here is expecting the shape to arrive as a table. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Parameters and headers **Query parameters** are the part after the `?`. Passing `params=` lets `requests` build them, which handles spaces and punctuation for you. **Headers** carry information about the request instead of information in it. This is where a key belongs, and section 32 is about why. """) return @app.cell def _(requests): prepared = requests.Request( "GET", "https://api.kraken.com/0/public/Ticker", params={"pair": "XBTUSD,ETHUSD", "note": "oim 7510 handbook"}, headers={"User-Agent": "oim7510-handbook"}, ).prepare() prepared.url return (prepared,) @app.cell def _(prepared): dict(prepared.headers) return @app.cell(hide_code=True) def _(mo, prepared): mo.md(f""" The URL came out as `{prepared.url}`. The comma between the two pairs became `%2C`, the spaces in the note became `+`, and `requests` joined the parameters with `&`. Nothing was sent here. `.prepare()` builds the request so you can look at it, which is a useful habit when a URL is not doing what you expect. ๐Ÿ‘‰ **Try it:** add a parameter whose value contains a `&` and read what `requests` did to it. Then work out what would have happened if you had built the URL by pasting strings together. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### A library can make the request for you Many services come with a Python library that sends these requests and hands back something more convenient. `yfinance` does it for Yahoo Finance's stock prices and gives you a DataFrame. It is an open-source project with no connection to Yahoo itself, and the one thing it does not hand you is the status code. The cell below asks for five days of Apple's prices, and then for a ticker that does not exist. """) return @app.cell def _(yf): try: apple_history = yf.Ticker("AAPL").history(period="5d") missing_history = yf.Ticker("NOTATICKERXYZ").history(period="5d") apple_rows = len(apple_history) missing_rows = len(missing_history) history_source = "live request" except Exception: apple_rows = 0 missing_rows = 0 history_source = "no network" if apple_rows == 0: apple_rows = 5 missing_rows = 0 history_source = "saved counts from 2026-09-15, Yahoo did not answer" f"{history_source} | AAPL {apple_rows} rows | NOTATICKERXYZ {missing_rows} rows" return apple_rows, missing_rows @app.cell(hide_code=True) def _(apple_rows, missing_rows, mo): mo.md(f""" Apple came back with **{apple_rows}** rows and the ticker that does not exist with **{missing_rows}**. Underneath, Yahoo answered that second request with HTTP 404. `yfinance` printed a message about it, returned an empty DataFrame and raised nothing, so your code carries on holding no data. **The trap.** The failure turns up later and somewhere else. Ask the empty frame for its latest closing price and you get this: ```python missing_history["Close"].iloc[-1] ``` ``` IndexError: single positional indexer is out-of-bounds ``` Nothing in that message mentions a ticker. A library that hides the status code leaves the checking to you, so look at `history.empty` before you index. ๐Ÿ‘‰ **Try it:** write a function that takes a ticker, fetches five days and returns the latest close, or returns `None` and prints a warning when the frame is empty. Run it on a real ticker and on a misspelled one. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" โœ” **Check your understanding** 1. A request returns status 200 and `.json()` works, and the answer is still missing. How can that happen, and where in the reply do you look? 2. You have a response from an API you have never seen before. Name the two lines you write before you index into it. 3. A price in the response is `"76386.40000"`. What does `price * 2` give you, and why does that not raise anything? 4. `yfinance` hands you an empty DataFrame for a ticker. What happened at the HTTP level, and why did your code not notice? 5. Where does an API key go on a request, and where does it not? """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## 32. API keys and environment variables A key is a password that identifies you to a service. Yours arrives by email from Babson. You do not create this key and it is not yours in the sense a password is, and every request made with it is billed to Babson and attributed to you. So it must never enter your repository. Your repositories in this course are published, so a committed key is a key that strangers can spend, and deleting it in a later commit does not remove it from the history. **This section is about code you write.** The key you pasted into marimo's settings screen is used by marimo's own chat panel, and code in a cell cannot see it. When your own code has to make the call, the key reaches it through the environment, and a `.env` file is what puts it there. ๐Ÿ“– **Used in:** How an LLM and an agent actually work """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### The environment is a set of names your program can read Every program starts with a set of name and value pairs handed to it by the operating system. `os.environ` is that set, and `os.getenv` reads one name out of it. """) return @app.cell def _(): import os key_present = os.getenv("ANTHROPIC_API_KEY") is not None f"ANTHROPIC_API_KEY is {'set' if key_present else 'not set'} in this environment" return (os,) @app.cell(hide_code=True) def _(mo): mo.md(r""" Note what that cell does not do. It never prints the value, and neither should any cell you write. A key printed into a notebook output is a key committed to your repository, because the output is saved in the file. ๐Ÿ‘‰ **Try it:** print `len(os.environ)` and then `sorted(os.environ)[:10]`. Your machine already carries dozens of these and none of them came from you. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### The files that hold a key Each of the three does one job, and none of them works without the other two. **`.env`** sits in your project folder and holds the real value. One name per line, no quotes, no spaces around the `=`: ``` ANTHROPIC_API_KEY=the-value-from-your-email ``` **`.gitignore`** tells git to pretend `.env` is not there, with the reason written where the next person will read it: ``` # Environment variables (contains API keys) .env ``` **Write `.gitignore` first.** It only affects files git is not tracking yet, so a `.env` committed even once stays tracked, and its value stays in the history whatever `.gitignore` says afterwards. **`.env.example`** is committed, and it is the same file with the values removed. It tells somebody who clones your work which names they have to supply: ``` # Copy this file to .env and fill in your own values. ANTHROPIC_API_KEY= ``` Setting up a clone is then one command: ``` cp .env.example .env ``` ๐Ÿ‘‰ **Try it:** create all three files in a scratch folder, put a made-up value in `.env`, run `git status`, and confirm that git offers you `.env.example` and says nothing about `.env`. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### `load_dotenv()` copies the file into the environment A `.env` file does nothing on its own. The `python-dotenv` library reads it and puts each name into `os.environ`, after which `os.getenv` finds it. ```python from dotenv import load_dotenv import os load_dotenv() api_key = os.getenv("ANTHROPIC_API_KEY") ``` Two lines of setup, and one line that gets the value. The Anthropic library reads `ANTHROPIC_API_KEY` by itself, so with those two lines run you can construct a client with no arguments at all and never touch the value: ```python import anthropic client = anthropic.Anthropic() ``` **The trap.** `load_dotenv()` returns `False` when it found no file, and it does not raise. So a missing `.env` gives you a program that runs, reads `None` for the key, and fails much later with an authentication error that says nothing about a missing file. ๐Ÿ‘‰ **Try it:** run `uv add python-dotenv`, then print what `load_dotenv()` returns in a folder with a `.env` and in a folder without one. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Fail loudly when the key is missing The fix for the trap above is four lines, and they belong before anything else runs. """) return @app.cell def _(os): def key_status(): """Report whether the key is available, without ever showing it.""" value = os.getenv("ANTHROPIC_API_KEY") if value is None: return "ANTHROPIC_API_KEY is missing. Check that .env exists and load_dotenv() ran." if not value.strip(): return "ANTHROPIC_API_KEY is set but empty. Check for a blank value in .env." return f"ANTHROPIC_API_KEY is set, {len(value)} characters." key_status() return @app.cell(hide_code=True) def _(mo): mo.md(r""" In real code the missing branch raises instead of returning a string, so that nothing downstream runs: ```python if not os.getenv("ANTHROPIC_API_KEY"): raise RuntimeError("ANTHROPIC_API_KEY is missing. Copy .env.example to .env.") ``` Reporting the length is safe and useful, because a key pasted with a trailing newline or half a key pasted from a wrapped email both show up as a wrong number. ๐Ÿ‘‰ **Try it:** call `key_status()` on your own machine before and after you create `.env`. Then paste a value with a space at the end and see whether the length tells you. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Where a key must never go **Not in `pyproject.toml`.** That file is part of your repository and it gets published. If an agent suggests putting a key there, decline. This is the one an agent gets wrong often enough that [Your AI Tools and Babson Key](https://oim6301.github.io/guides/babson-ai-key/) warns about it separately. **Not in a notebook cell.** `client = anthropic.Anthropic(api_key="sk-...")` runs perfectly and commits the key with the notebook. **Not in a URL.** Last year's weather example put its key in a query string, because that is what that particular service wanted. A model API wants it in a header, and the library puts it there for you. Query strings turn up in server logs and browser history; headers do not. **Not in a printed output.** Including in an error message you paste into a chat when asking for help. **When a key does leak, the fix is a new key.** Rewriting git history is slow, incomplete, and beside the point once the value has been public for any length of time. Mail ai@babson.edu, get a replacement, and put the old one out of use. ๐Ÿ‘‰ **Try it:** open the last repository you pushed and run `git log -p | grep -i "api_key"`. Do this before you need to. โœ” **Check your understanding** 1. You committed a key, noticed an hour later, and deleted the line in a new commit. What is still true? 2. `.env` is gitignored and `.env.example` is committed. What does each one contain, and why is that the way round it is? 3. Your code runs and gets an authentication error. Name two causes that have nothing to do with the key being wrong. 4. Why does a key belong in a header instead of in the URL? """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## 33. What a request to a model contains A request to a model is a dictionary with four things in it, and once you have seen them the rest of this chapter is variations on the same shape. There is no session, no connection, and nothing on the far side that remembers you. You send everything, every time. ๐Ÿ“– **Used in:** How an LLM and an agent actually work """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### The parts of a request to a model `model` says which model. `max_tokens` caps how long the reply may be, and it is **required**: leave it out and the request is rejected. `system` sets standing instructions. `messages` is the conversation. Here is the request as a plain dictionary, so you can read it before you send one. """) return @app.cell def _(): request_body = { "model": "MODEL", "max_tokens": 1024, "system": "You are helping an MBA student check a freight calculation. Be brief.", "messages": [ {"role": "user", "content": "Order 10248 had freight of 16.75. Is that high?"}, ], } sorted(request_body) return (request_body,) @app.cell(hide_code=True) def _(mo, request_body): mo.md(f""" Four keys, and `messages` holds **{len(request_body["messages"])}** entry so far. The library sends this to the server as JSON, so what goes over the wire is the dictionary you just read. `"MODEL"` is a placeholder. The real model id comes from the email that issued your key, and it goes in a name at the top of your file so that changing it is one edit: ```python MODEL = "the model id from your email" ``` ๐Ÿ‘‰ **Try it:** build the same dictionary with your own question in it, then run `json.dumps(request_body, indent=2)` and read what the server will actually receive. ๐Ÿ‘‰ **Then send it**, with the key from your Babson email, and read the reply's own shape rather than this page's description of it. Your base URL carries no `/v1`, so add `/v1/messages` to it. `-i` prints the status line; `-s` hides the errors worth seeing. ```bash curl -i "$BASE/v1/messages" -H "x-api-key: $KEY" -H "anthropic-version: 2023-06-01" -H "content-type: application/json" -d '{{"model":"claude-sonnet-5","max_tokens":16,"system":"Answer in one word.","messages":[{{"role":"user","content":"hi"}}]}}' ``` Three things to look for, because they are what this section claims: `system` went in at the top level and not inside `messages`, `content` came back as a **list** of blocks rather than a string, and there is a `usage` object counting both directions. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### `system` is a top-level parameter In the Anthropic API, `system` sits beside `model` and `max_tokens`. It is not an entry in `messages`. ```python response = client.messages.create( model=MODEL, max_tokens=1024, system="You are helping an MBA student check a freight calculation. Be brief.", messages=[{"role": "user", "content": "Order 10248 had freight of 16.75. Is that high?"}], ) ``` **The trap, and it is the one an agent will hand you.** Most example code on the internet was written for a different provider, where the system prompt is the first entry in `messages` with `"role": "system"`. Ported across without changing it, the request is still valid and still runs. The instruction is now a user turn, the model treats it as something you wrote, the standing rule you meant it to be is gone, and nothing warns you. ๐Ÿ‘‰ **Try it:** ask your agent to write you a Claude API call. Look at where it put the system prompt before you look at anything else. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### The shape of `messages` Every entry has a `role` and a `content`. The role is `user` for your turns and `assistant` for the model's, and those are the literal strings the API uses. The first entry has to be a `user` turn. After that they alternate. """) return @app.cell def _(): three_turns = [ {"role": "user", "content": "Order 10248 had freight of 16.75. Is that high?"}, {"role": "assistant", "content": "It depends what you compare it with."}, {"role": "user", "content": "Compare it with the other four orders that week."}, ] [entry["role"] for entry in three_turns] return (three_turns,) @app.cell(hide_code=True) def _(mo, three_turns): mo.md(f""" The list holds three entries, with roles **{", ".join(e["role"] for e in three_turns)}**. That list is the whole conversation, it lives in your program, and section 34 is about what happens when you forget to add to it. ๐Ÿ‘‰ **Try it:** build a list where two `user` turns sit next to each other. It is legal; the API joins them into one turn. Then build one that starts with an `assistant` turn and find out what the library says. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### What comes back is a list of blocks The reply is not a string. `response.content` is a **list**, and each entry has a `type`. Today it is usually one text block; with a tool in play there are two, and section 36 is where that matters. Below is the JSON the API returns, built as a dictionary so you can read it with code you already have. The library hands you objects with the same field names, so `reply["stop_reason"]` here is `response.stop_reason` there. """) return @app.cell def _(): reply = { "id": "msg_example", "type": "message", "role": "assistant", "content": [ { "type": "text", "text": "Freight of 32.38 is the highest of the five orders placed that week. The others range from 3.05 to 51.30, so it is high but not an outlier.", } ], "stop_reason": "end_turn", "stop_details": None, "usage": {"input_tokens": 1284, "output_tokens": 412}, } sorted(reply) return (reply,) @app.cell def _(reply): def first_text(message): """Return the text of the first text block, or None when there is not one.""" for block in message["content"]: if block["type"] == "text": return block["text"] return None first_text(reply) return @app.cell(hide_code=True) def _(mo, reply): mo.md(f""" The reply carries **{len(reply["content"])}** block, of type `{reply["content"][0]["type"]}`, and it stopped because of `{reply["stop_reason"]}`. `usage` says the request cost **{reply["usage"]["input_tokens"]}** input tokens and **{reply["usage"]["output_tokens"]}** output tokens, which section 38 turns into money. **The trap.** `response.content[0].text` works most of the time and is wrong. It assumes there is at least one block and that the first one is text, and both of those stop being true the moment you add a tool. Loop over the blocks and check the type, the way `first_text` does. ๐Ÿ‘‰ **Try it:** add a second block of type `tool_use` to the front of `reply["content"]` and run `first_text` again. Then run `reply["content"][0]["text"]` and read the error. โœ” **Check your understanding** 1. Name the four parts of a request and state which one is required and easy to forget. 2. Somebody hands you code with `{{"role": "system", ...}}` as the first entry in `messages`. What happens when you run it against the Anthropic API? 3. `response.content` is a list. Give a case where it has two entries. 4. Where does `stop_reason` live, and what is it for? """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## 34. The API is stateless **The API is stateless.** Nothing on the server keeps track of your conversation between one request and the next. So when a model appears to remember what you wrote three turns ago, it is because your program sent all three turns again. The memory is a Python list, it lives on your machine, and you are responsible for it. ๐Ÿ“– **Used in:** How an LLM and an agent actually work """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### The loop that works Here is the whole shape. Line by line: read a question, add it to the list, send **the entire list**, add the reply to the list, print it. ```python messages = [] while True: user_input = input("You: ") if user_input.lower() in ["quit", "exit", "bye"]: break messages.append({"role": "user", "content": user_input}) response = client.messages.create( model=MODEL, max_tokens=1024, system="You are a Python expert.", messages=messages, ) answer = next(b.text for b in response.content if b.type == "text") messages.append({"role": "assistant", "content": answer}) # <- this line print(f"Model: {answer}") ``` The marked line is the one that matters. Delete it and the program runs, prints, never raises, never warns, and the model forgets everything after every turn. ๐Ÿ‘‰ **Try it:** copy this into a file, run it with your key, and ask it two questions where the second one depends on the first. Then delete the marked line and ask the same two questions. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### The conversations side by side A model call needs a key, so the cells below use a stand-in that gives back a fixed reply and reports how many turns it was handed. That is enough to show the difference, because the difference is entirely on your side of the wire. """) return @app.cell def _(): def stand_in_reply(sent): """Stand in for a model call. Report how many turns the request carried.""" return f"(a reply, written after reading {len(sent)} turns)" questions = [ "My name is Dana and I work in logistics.", "Which order that week had the highest freight?", "What was my name again?", ] return questions, stand_in_reply @app.cell def _(questions, stand_in_reply): messages = [] for question in questions: messages.append({"role": "user", "content": question}) answer = stand_in_reply(messages) messages.append({"role": "assistant", "content": answer}) len(messages) return (messages,) @app.cell def _(questions, stand_in_reply): forgetful = [] for forgetful_question in questions: forgetful.append({"role": "user", "content": forgetful_question}) stand_in_reply(forgetful) # the reply is never appended len(forgetful) return (forgetful,) @app.cell(hide_code=True) def _(forgetful, messages, mo): mo.md(f""" After three questions the working version holds **{len(messages)}** entries and the broken one holds **{len(forgetful)}**. The last request in the working version carried **{len(messages) - 1}** turns; in the broken one it carried **{len(forgetful)}**, all of them questions, with no sign that anything was ever answered. Neither program raised anything. Neither printed a warning. The only visible difference is that the third answer is wrong, and only if you happened to ask a question whose answer depended on the first two. ๐Ÿ‘‰ **Try it:** print both lists. Then work out what the model saw on the third request in the broken version, and what it would have to guess. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Sending everything every time has a price Each turn resends every turn before it. So the tenth request is not the same size as the first. """) return @app.cell def _(messages): def characters_sent(conversation): """Total characters sent, if every turn resends the whole conversation.""" running = 0 for turn in range(1, len(conversation) + 1): running += sum(len(entry["content"]) for entry in conversation[:turn]) return running characters_sent(messages), sum(len(e["content"]) for e in messages) return (characters_sent,) @app.cell(hide_code=True) def _(characters_sent, messages, mo): mo.md(f""" The conversation is **{sum(len(e["content"]) for e in messages)}** characters long, and holding it for six turns meant sending **{characters_sent(messages)}** characters, which is **{characters_sent(messages) / sum(len(e["content"]) for e in messages):.1f}** times as many. That ratio grows with the length of the conversation, and section 38 puts a price on it. ๐Ÿ‘‰ **Try it:** run `characters_sent` on a list of 20 turns and compare it with the length of the conversation. Then work out roughly what the ratio would be at 50 turns. โœ” **Check your understanding** 1. The model answered a question that depended on something you wrote two turns ago. Where was that information stored? 2. You delete the line that appends the reply. Name everything that goes wrong, and everything that reports it. 3. Why does turn 10 of a conversation cost more than turn 1, given that you typed the same amount? 4. Two people run your chat program at once. How many conversations does the server keep track of? """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## 35. The same prompt twice Send a model the same request twice and you can get two different answers. Two wordings of one answer is the harmless version of this. The version that costs you something is two different numbers, or two different thresholds, or two decisions that contradict each other. This is a property of how these models produce text, and no setting available to you removes it. What you can do about it is run the request more than once and compare the answers. ๐Ÿ“– **Used in:** How an LLM and an agent actually work """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### The same code gives a different total Before the model, here is the same behavior in three lines you can read completely. Roll a die 100 times. """) return @app.cell def _(random): def roll(times): """Roll a six-sided die and report the total and the average.""" rolls = [random.randint(1, 6) for _ in range(times)] return sum(rolls), sum(rolls) / times roll(100) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Run that cell again and the total changes. Run it 10 times and the average stays near 3.5 while the total keeps moving. The code did not change and neither did the input, and you can still rely on the average. A model is not a die and the analogy stops quickly. What carries over is the working habit. One run tells you very little, and several runs tell you what is stable. ๐Ÿ‘‰ **Try it:** call `roll(10)` five times and `roll(10000)` five times. Which average moves more, and why does that matter for how many times you should run a prompt? """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### There is no setting that makes it repeat `temperature` is the parameter people reach for. It takes a number from 0 to 1 and low values make the model pick the likeliest next piece of text more often. Two things about it, and the first one is permanent. **Temperature 0 does not make output deterministic.** It narrows the spread. Identical requests at temperature 0 still come back different, and any workflow built on the assumption that they will not is built on sand. **On some newer models the parameter is gone.** Sending `temperature` to a model that no longer takes it returns a 400, and the request never runs: ``` anthropic.BadRequestError: Error code: 400 - {'type': 'error', 'error': {'type': 'invalid_request_error', 'message': 'temperature: Extra inputs are not permitted'}} ``` Other models still accept it. Whether the model behind your key does is worth one request to find out, and the answer changes nothing about the paragraph above. ๐Ÿ‘‰ **Try it:** send the same request twice with your key and `diff` the two answers. Then, if your model accepts `temperature`, set it to 0 and do it again. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Reading the spread across runs Here are five replies to one request, recorded from five runs of *write Python that tiers an order value into A, B or C*. Nothing about the request changed between them. """) return @app.cell def _(): five_runs = [ "Orders of 1000 or more are tier A, 500 or more are tier B, and below 500 is tier C.", "Tier A starts at 1000. Tier B covers 250 up to 1000. Everything under 250 is tier C.", "I used 1000 and 500 as the cutoffs, so A is 1000 and above, B is 500 and above.", "Tier A is above 5000, tier B is above 1000, and the rest are tier C.", "A for 1000 or more, B for 500 or more, C otherwise.", ] len(five_runs) return (five_runs,) @app.cell def _(five_runs, re): thresholds = [tuple(sorted({int(n) for n in re.findall(r"\d+", run)})) for run in five_runs] distinct_sets = sorted(set(thresholds)) top_boundaries = sorted({max(cutoffs) for cutoffs in thresholds}) thresholds, distinct_sets, top_boundaries return distinct_sets, top_boundaries @app.cell(hide_code=True) def _(distinct_sets, mo, top_boundaries): mo.md(f""" Five runs produced **{len(distinct_sets)}** different sets of cutoffs, and the highest boundary alone took **{len(top_boundaries)}** different values: {" and ".join(f"{value:,}" for value in top_boundaries)}. One run of five put tier A at {max(top_boundaries):,}, which is **{max(top_boundaries) // min(top_boundaries)}** times the value the other four chose and would be invisible if you had run it once. **The trap.** Reading one output and finding it reasonable is not a check. The output you happened to get is one draw, and the run that disagrees with the other four is the one you needed to see. ๐Ÿ‘‰ **Try it:** send *write Python that tiers an order value into A, B or C* five times with your key, pull the numbers out of each answer, and count the distinct sets. Then look at whether the boundary is `>` or `>=` in each one. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### What to verify Verify the number, and let the wording vary. Five answers that word the same threshold differently are fine. Five answers with three different thresholds mean the model was filling in a decision you never made, and the fix is to make it and state it in the request. Every deliverable in this course carries a **How I know these numbers are right** section, and that section needs a check that runs. A number you can recompute is a number that survives the model producing a different answer tomorrow. ๐Ÿ‘‰ **Try it:** take one of the five replies above, write the `assert` that would have caught the 5000 version, and state what you had to decide in order to write it. โœ” **Check your understanding** 1. You set `temperature` to 0 and get the same answer twice. What have you established? 2. Your agent gave you a working function and you read it and it looked right. What have you checked, and what have you not? 3. Five runs give five wordings of the same threshold. Is that a problem? 4. Name something about a model's answer that is worth verifying, and something that is not worth arguing with. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## 36. Tools A model can read and write text. It cannot look anything up, run anything, or touch anything. A **tool** is how you close that gap, and the mechanism is smaller than it sounds. You describe a function you already have, the model asks you to call it, you call it, and you send back the answer. The model never runs your code. It asks. Everything that happens, happens because your program did it. ๐Ÿ“– **Used in:** How an LLM and an agent actually work """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### The function first Nothing about this function knows a model exists. """) return @app.cell def _(): ORDER_FREIGHT = {10248: 32.38, 10249: 11.61, 10250: 65.83, 10251: 41.34, 10252: 51.30} def order_freight(order_id): """Return the freight charge for one order, or None when the order is unknown.""" return ORDER_FREIGHT.get(order_id) order_freight(10250) return (order_freight,) @app.cell(hide_code=True) def _(mo): mo.md(r""" ๐Ÿ‘‰ **Try it:** call it with an order id that is not in the dictionary. Chapter B section 15 is why the answer is `None` instead of a `KeyError`, and that choice matters here, because a tool that raises takes the whole loop down with it. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### The tool definition is a description of it Three keys. `name` has to match what your code dispatches on. `description` is what the model reads to decide whether this is the right tool, so it is prose and it is worth writing properly. `input_schema` is JSON Schema, which is a dictionary describing the arguments. """) return @app.cell def _(): freight_tool = { "name": "order_freight", "description": "Look up the freight charge for one order by its order id. Returns the charge in dollars, or nothing when the order is unknown.", "input_schema": { "type": "object", "properties": { "order_id": { "type": "integer", "description": "The order id, for example 10248.", }, }, "required": ["order_id"], }, } sorted(freight_tool), freight_tool["input_schema"]["required"] return (freight_tool,) @app.cell(hide_code=True) def _(freight_tool, mo): mo.md(f""" The definition goes on the request as `tools=[freight_tool]`, beside `model`, `max_tokens`, `system` and `messages`: ```python response = client.messages.create( model=MODEL, max_tokens=1024, tools=[freight_tool], messages=[{{"role": "user", "content": "What was the freight on order 10250?"}}], ) ``` **The description is the whole interface.** The model chooses this tool on the strength of **{len(freight_tool["description"])}** characters of prose and nothing else. A tool that never gets called and a tool that gets called for the wrong thing are usually both description problems. ๐Ÿ‘‰ **Try it:** write a definition for a second tool that takes two arguments, one of them optional. Look at what changes in `required`. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### The model asks in a block of its own When the model wants the tool, the reply comes back with `stop_reason` of `tool_use` and a block of type `tool_use` in the content list. Here is that reply. """) return @app.cell def _(): tool_request = { "id": "msg_example_tool", "role": "assistant", "content": [ {"type": "text", "text": "Let me look that up."}, { "type": "tool_use", "id": "toolu_example", "name": "order_freight", "input": {"order_id": 10250}, }, ], "stop_reason": "tool_use", "usage": {"input_tokens": 512, "output_tokens": 74}, } [block["type"] for block in tool_request["content"]] return (tool_request,) @app.cell(hide_code=True) def _(mo, tool_request): mo.md(f""" Two blocks, and `content[0]` is text, which is why section 33 said not to reach for `content[0]["text"]` as if it were the answer. The `tool_use` block carries three things you need: an `id` of `{tool_request["content"][1]["id"]}`, a `name` of `{tool_request["content"][1]["name"]}`, and an `input` dictionary holding `{tool_request["content"][1]["input"]}`. ๐Ÿ‘‰ **Try it:** write the expression that pulls out every `tool_use` block from a reply, without assuming how many there are or where they sit. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Running the tool the model chose The model gave you a name and a dictionary of arguments. Your code turns that into a call. """) return @app.cell def _(order_freight, tool_request): TOOLS = {"order_freight": order_freight} def run_tool(block): """Run the tool the model asked for and return its result as text.""" function = TOOLS.get(block["name"]) if function is None: return f"No tool named {block['name']}." return str(function(**block["input"])) tool_answer = run_tool(tool_request["content"][1]) tool_answer return run_tool, tool_answer @app.cell(hide_code=True) def _(mo): mo.md(r""" `**block["input"]` unpacks the dictionary into keyword arguments, so `{"order_id": 10250}` becomes `order_freight(order_id=10250)`. That works because the schema named the argument `order_id` and so does the function. When those two names drift apart you get a `TypeError` about an unexpected keyword argument, and the schema is where to look. ๐Ÿ‘‰ **Try it:** change the schema's property name to `order`. A model fills in the names the schema gives it, so change the `input` in `tool_request` to `{"order": 10250}` as well, and leave the function alone. Read the error and decide which of the two you would fix. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### You send the result back as a user turn The result goes into the conversation as a `tool_result` block inside a message with role `user`. That surprises people, and it is consistent. Everything that comes from your side of the wire is a user turn, including a number your own function produced. Two rules. `tool_use_id` has to match the `id` from the request. And every result for one reply goes back in one message, however many tools were asked for. """) return @app.cell def _(tool_answer, tool_request): tool_conversation = [ {"role": "user", "content": "What was the freight on order 10250?"}, {"role": "assistant", "content": tool_request["content"]}, { "role": "user", "content": [ { "type": "tool_result", "tool_use_id": tool_request["content"][1]["id"], "content": tool_answer, } ], }, ] [entry["role"] for entry in tool_conversation] return (tool_conversation,) @app.cell(hide_code=True) def _(mo, tool_conversation): mo.md(f""" The list holds three entries, with roles **{", ".join(e["role"] for e in tool_conversation)}**. It goes back to the model as `messages`, and the next reply is the answer to the original question, written with the number in hand. **The trap, and it fails silently.** If your code reads the `tool_use` block and never sends a `tool_result` back, nothing raises. The model asked, you did not answer, and the conversation stops with an unanswered request in it. The tell is a reply whose `stop_reason` is `tool_use` and a program that printed nothing. **When the tool fails**, send the failure back. Add `"is_error": True` to the block and put the message in `content`. The model can work with a tool that said no; it cannot work with silence. ๐Ÿ‘‰ **Try it:** build the `tool_result` block for a failed call, with `is_error` set. Then build the message that would carry two results at once. โœ” **Check your understanding** 1. The model decided to use your tool. What ran on the server, and what ran on your machine? 2. Your tool is never chosen. Name the first thing to look at. 3. What has to match between the `tool_use` block and the `tool_result` block, and what happens when it does not? 4. Why is a tool result carried in a message with role `user`? """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## 37. The agent loop Section 36 did one round trip by hand. An agent is that round trip in a `while`, run until the model stops asking for tools. That loop is the entire difference between a chat window and an agent, and it is under 20 lines. It also raises the one question the API cannot answer for you. When does it stop? ๐Ÿ“– **Used in:** How an LLM and an agent actually work ยท Building your product """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### `stop_reason` is how the model tells you what it wants Every reply carries one. Four of them matter here. | `stop_reason` | what it means | what you do | |---|---|---| | `end_turn` | the model finished | stop, and read the answer | | `tool_use` | it wants a tool run | run it, send the result, go again | | `max_tokens` | it hit the ceiling you set and was cut off | the answer is incomplete, raise `max_tokens` and retry | | `refusal` | it declined | read `stop_details`, do not retry the same request | **`max_tokens` is the one that catches people.** The reply arrives, it reads like an answer, and it stopped mid-sentence because the cap was too low. Nothing raises. Check `stop_reason` before you use a reply for anything. ๐Ÿ‘‰ **Try it:** send a request with `max_tokens=20` and a question that needs a paragraph. Print `response.stop_reason` and then print the text. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### The loop Here it is against a real client. It is section 36 four times over. ```python messages = [{"role": "user", "content": question}] while True: response = client.messages.create( model=MODEL, max_tokens=1024, tools=[freight_tool], messages=messages, ) if response.stop_reason != "tool_use": break messages.append({"role": "assistant", "content": response.content}) results = [ {"type": "tool_result", "tool_use_id": b.id, "content": run_tool(b)} for b in response.content if b.type == "tool_use" ] messages.append({"role": "user", "content": results}) ``` Below, the same loop runs against a stand-in that returns replies from a list, so you can watch it go round without a key. """) return @app.cell def _(run_tool): def build_stand_in(script): """Return a function that hands back the next scripted reply each time it is called.""" remaining = list(script) def send(_sent): return remaining.pop(0) if remaining else {"stop_reason": "end_turn", "content": []} return send def agent_loop(send, question, limit): """Run the tool loop. Stop on end_turn, or at the limit, and report which.""" conversation = [{"role": "user", "content": question}] log = [] for step in range(1, limit + 1): response = send(conversation) log.append( f"step {step}: entries sent {len(conversation)}, " f"stop_reason {response['stop_reason']}" ) if response["stop_reason"] != "tool_use": return conversation, log, "the model said it was done" conversation.append({"role": "assistant", "content": response["content"]}) results = [ {"type": "tool_result", "tool_use_id": b["id"], "content": run_tool(b)} for b in response["content"] if b["type"] == "tool_use" ] conversation.append({"role": "user", "content": results}) return conversation, log, f"the limit of {limit} steps was reached" return agent_loop, build_stand_in @app.cell def _(): def asks_for(order_id): """A scripted reply in which the model asks for one order's freight.""" return { "stop_reason": "tool_use", "content": [ { "type": "tool_use", "id": f"toolu_{order_id}", "name": "order_freight", "input": {"order_id": order_id}, } ], } FINISHES = { "stop_reason": "end_turn", "content": [{"type": "text", "text": "Order 10250 is the highest of the three at 65.83."}], } return FINISHES, asks_for @app.cell def _(FINISHES, agent_loop, asks_for, build_stand_in): good_conversation, good_log, good_reason = agent_loop( build_stand_in([asks_for(10248), asks_for(10249), asks_for(10250), FINISHES]), "Which of orders 10248, 10249 and 10250 had the highest freight?", limit=10, ) good_log return good_conversation, good_reason @app.cell(hide_code=True) def _(good_conversation, good_reason, mo): mo.md(f""" Four steps, three of them tool calls, and it stopped because **{good_reason}**. The conversation ended up **{len(good_conversation)}** entries long from one question, because every tool call adds two. ๐Ÿ‘‰ **Try it:** change the script so the model asks for a fourth order and then finishes. Predict the number of entries before you run it. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Print what the model receives The model sees the list you send and nothing else. So the most useful line in an agent loop is one that prints that list before each call. Against a real client it is `print(messages)`, directly above `client.messages.create(...)`. The output is long, and it is everything the model will see. Here is the conversation from the run above, one entry per line. """) return @app.cell def _(good_conversation): def describe(entry): """One line for one entry in the conversation: who sent it and what it carries.""" if isinstance(entry["content"], str): return f"{entry['role']}: {entry['content']}" parts = [] for block in entry["content"]: if block["type"] == "text": parts.append(block["text"]) elif block["type"] == "tool_use": parts.append(f"asks for {block['name']}({block['input']})") elif block["type"] == "tool_result": parts.append(f"tool returned {block['content']}") return f"{entry['role']}: " + " | ".join(parts) [describe(entry) for entry in good_conversation] return (describe,) @app.cell(hide_code=True) def _(mo): mo.md(r""" Step 1 was sent the first line. Step 2 was sent the first three, and step 4 was sent all seven. **Every call resends everything before it**, which is why the log above shows the entry count climbing by two per tool call. The tool results arrive as `user` lines, as section 36 said they would. ๐Ÿ‘‰ **Try it:** give `describe` one more `elif` that marks a `tool_result` carrying `is_error`. Then build a conversation with a failed call in it and read the line. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### The stop condition the API will not give you The loop above stops when the model says it is done. Nothing guarantees the model will say it is done. A tool that keeps returning something unsatisfying, a question with no answer in the data, an agent that decides to look one more thing up. Any of those turns `while True` into a loop that runs until your budget stops it. So the limit is yours to set. Here is the identical loop against a model that never finishes. """) return @app.cell def _(agent_loop, asks_for, build_stand_in): stuck_conversation, stuck_log, stuck_reason = agent_loop( build_stand_in([asks_for(10248)] * 50), "Which of orders 10248, 10249 and 10250 had the highest freight?", limit=5, ) stuck_log return stuck_conversation, stuck_reason @app.cell(hide_code=True) def _(good_conversation, mo, stuck_conversation, stuck_reason): mo.md(f""" It stopped because **{stuck_reason}**, after building a conversation of **{len(stuck_conversation)}** entries against **{len(good_conversation)}** for the working one. Without the limit that loop would have run 50 times, and against a live model it would have run until something else stopped it. **A step limit is what keeps a runaway loop off your budget.** [Your AI Tools and Babson Key](https://oim6301.github.io/guides/babson-ai-key/) says the monthly budget covers this course comfortably, and it stops covering it when an agent is left rerunning a failing step 20 times with nobody reading the error. A `for` with a ceiling instead of a `while True` is the whole defense. **And every step costs more than the one before it**, because the conversation it resends is longer. Step 5 of a stuck loop is the most expensive one in it. ๐Ÿ‘‰ **Try it:** raise the limit to 20 and watch the conversation length. Then add a counter to the loop that stops when the same tool is called with the same input twice in a row, and name what that would have caught. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### A check you write decides when it is done `end_turn` means the model judged its own answer finished. That is a claim, and the loop can test it before accepting it. Write a function that reads the answer and returns a problem, or `None` when there is none. Only the branch for a finished reply changes: ```python if response["stop_reason"] != "tool_use": answer = " ".join(b["text"] for b in response["content"] if b["type"] == "text") problem = find_problem(answer) if problem is None: return conversation, log, "the check passed" conversation.append({"role": "assistant", "content": response["content"]}) conversation.append( {"role": "user", "content": f"Your answer did not pass my check: {problem}"} ) continue ``` A problem goes back to the model as a user turn, and `continue` sends the loop round again. So the loop stops in one of two ways: the check passed, or the limit was reached. The model reporting that it is done no longer ends it. For the freight question, a check can recompute the answer with the tool itself: ```python def find_problem(answer): highest = max([10248, 10249, 10250], key=order_freight) if str(highest) not in answer: return f"it does not name order {highest}" return None ``` A check like this needs a question whose answer you already know, which is how you test an agent before you trust it with a question you do not. ๐Ÿ‘‰ **Try it:** add `find_problem` as a fourth parameter of `agent_loop` and put the branch above in it. Script a stand-in whose first reply names order 10249, and read the log. After section 38, add a second ceiling: stop when the tokens in `usage` add up past a budget you choose. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" > **Advanced** > > The Anthropic library ships this loop, so you can hand it your functions and let it > run: `client.beta.messages.tool_runner(...)`, with each tool marked by a `@beta_tool` > decorator that builds the schema from the function signature. It is in beta, and the > reason to write the loop by hand at least once is that the runner's stopping behavior > is then something you can reason about instead of something you hope about. โœ” **Check your understanding** 1. What is the difference between a chat program and an agent, in code? 2. A reply comes back with `stop_reason` of `max_tokens`. Is the answer usable? 3. Your loop has been going for 40 steps. Name two things that are growing and one thing that is not. 4. Where does the stop condition come from, and what happens when you leave it out? 5. The model replied with `end_turn`. What has to be true before you treat its answer as finished, and where in the loop does that get tested? """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## 38. Tokens and cost A model does not read characters or words. It reads **tokens**, which are pieces of text of roughly a few characters, and it is billed by the token in both directions. Two numbers come back on every reply. ๐Ÿ“– **Used in:** How an LLM and an agent actually work ยท Building your product """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### `usage` is on every reply You already have it. The reply built in section 33 carries a `usage` dictionary with two counts in it. """) return @app.cell def _(reply): reply["usage"] return @app.cell(hide_code=True) def _(mo, reply): mo.md(f""" **{reply["usage"]["input_tokens"]}** tokens went in and **{reply["usage"]["output_tokens"]}** came out. Input is everything you sent: the system prompt, the tool definitions, and every turn of the conversation. Output is what the model wrote. ๐Ÿ‘‰ **Try it:** send any request with your key and print `response.usage`. Then send the same request with a longer system prompt and print it again. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Counting tokens before you send A token count is a property of the model's tokenizer, so the only accurate count comes from the model's own API. One call, and it does not run the model: ```python count = client.messages.count_tokens( model=MODEL, system=system_prompt, messages=messages, ) print(count.input_tokens) ``` **Do not use `tiktoken` for this.** It is a different company's tokenizer, it undercounts Claude by roughly 15 to 20 percent on ordinary text, and by more on code. A cost estimate built on it is wrong in the direction that costs you money. Rules of thumb for characters per token float around and none of them are worth memorizing, because the ratio depends on the text. Measure your own. ๐Ÿ‘‰ **Try it:** run `count_tokens` on a paragraph of your own writing, and again on 20 lines of your own Python. Divide each by its character count. The two ratios will not match, and that is the point. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Tokens times a price Prices are quoted per million tokens, and output costs several times what input does. Put both in names at the top so that a price change is one edit. """) return @app.cell def _(reply): INPUT_PER_MILLION = 2.00 OUTPUT_PER_MILLION = 10.00 def cost_of(usage): """Cost in dollars for one reply, from its usage counts.""" return ( usage["input_tokens"] / 1_000_000 * INPUT_PER_MILLION + usage["output_tokens"] / 1_000_000 * OUTPUT_PER_MILLION ) cost_of(reply["usage"]) return INPUT_PER_MILLION, OUTPUT_PER_MILLION, cost_of @app.cell(hide_code=True) def _(INPUT_PER_MILLION, OUTPUT_PER_MILLION, cost_of, mo, reply): mo.md(f""" That request cost **{cost_of(reply["usage"]):.5f}** dollars, which is under a cent. The two prices used are **{INPUT_PER_MILLION:.2f}** and **{OUTPUT_PER_MILLION:.2f}** dollars per million, published for the model this course runs on and checked on 2026-09-14. Prices move. Put the current ones in those two names before you trust a total. **`max_tokens` is a ceiling and not a target.** Setting it to 4096 does not cost you 4096 output tokens. You are billed for what came back. Set it high enough that answers are not cut off, and use `stop_reason` to find out when it was too low. ๐Ÿ‘‰ **Try it:** work out what 1,000 requests of this size would cost. Then work out how much of that is output. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### A conversation costs more than the sum of its turns Section 34 established that every request resends the whole conversation. Here is what that does to a bill. Suppose each turn adds 900 tokens, and you have 10 turns. """) return @app.cell def _(): TOKENS_PER_TURN = 900 TURNS = 10 naive_input = TOKENS_PER_TURN * TURNS actual_input = sum(TOKENS_PER_TURN * turn for turn in range(1, TURNS + 1)) naive_input, actual_input, actual_input / naive_input return TURNS, actual_input, naive_input @app.cell(hide_code=True) def _(INPUT_PER_MILLION, TURNS, actual_input, mo, naive_input): mo.md(f""" Typing **{naive_input:,}** tokens of conversation sends **{actual_input:,}** of them, which is **{actual_input / naive_input:.1f}** times as many, and costs **{actual_input / 1_000_000 * INPUT_PER_MILLION:.4f}** dollars against **{naive_input / 1_000_000 * INPUT_PER_MILLION:.4f}**. The ratio is not a constant. At {TURNS} turns it is **{actual_input / naive_input:.1f}**; the longer the conversation runs the worse it gets, because turn 40 resends 39 turns. Long agent runs are expensive for this reason before anything goes wrong. ๐Ÿ‘‰ **Try it:** change `TURNS` to 40 and read the ratio. Then find the number of turns at which the conversation costs 10 times what typing it would suggest. โœ” **Check your understanding** 1. You raised `max_tokens` from 1024 to 8192 and sent the same request. What happened to the cost? 2. Name three things that count as input tokens on a request with a tool in it. 3. Why does a 40-turn conversation cost more than four 10-turn conversations carrying the same text? 4. Somebody estimates their bill with `tiktoken` and gets 5 dollars. Which direction is the real number in? """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## 39. Embeddings and similarity > **Advanced** > > Nothing in this chapter, and nothing in any assignment, needs this section. It is > here because *find the documents that are about the same thing as this one* is a > question people expect a model to answer, and the machinery underneath is two ideas > you can read completely. An **embedding** is a list of numbers standing for a piece of text, arranged so that texts about the same thing get similar lists. Once text is numbers, *similar* becomes arithmetic. To see the arithmetic without a model, the section below builds the numbers by counting words. That is a poor embedding and a perfectly good demonstration, because the similarity step is identical either way. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Text becomes a vector Take the list of every word that appears anywhere, then count each word in each sentence. Each sentence becomes one row of counts. """) return @app.cell def _(): SENTENCES = [ "the order shipped late", "the shipment arrived late", "the order shipped on time", "freight charges rose in July", ] VOCABULARY = sorted({word for sentence in SENTENCES for word in sentence.split()}) def to_vector(sentence): """Count each vocabulary word in one sentence.""" words = sentence.split() return [words.count(term) for term in VOCABULARY] VOCABULARY return SENTENCES, VOCABULARY, to_vector @app.cell def _(SENTENCES, to_vector): vectors = [to_vector(sentence) for sentence in SENTENCES] vectors[0], vectors[1] return (vectors,) @app.cell(hide_code=True) def _(VOCABULARY, mo, vectors): mo.md(f""" The vocabulary holds **{len(VOCABULARY)}** words, so every sentence became a list of **{len(vectors[0])}** numbers, most of them zero. The first two sentences differ in **{sum(1 for x, y in zip(vectors[0], vectors[1]) if x != y)}** positions, and the two words they share are *the* and *late*. ๐Ÿ‘‰ **Try it:** add a sentence of your own and print the vocabulary length before and after. Then explain what happens to the length of every existing vector. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Cosine similarity The measure everybody uses is the cosine of the angle between two vectors. It runs from 0 for nothing in common to 1 for pointing the same way, and it ignores length, which is what stops a long document from looking similar to everything. Multiply the pairs and add them up, then divide by the two lengths. """) return @app.cell def _(): def cosine(a, b): """Cosine similarity of two vectors of the same length.""" dot = sum(x * y for x, y in zip(a, b)) return dot / ((sum(x * x for x in a) ** 0.5) * (sum(y * y for y in b) ** 0.5)) cosine([1, 1, 0], [1, 1, 0]), cosine([1, 0, 0], [0, 1, 0]) return (cosine,) @app.cell def _(SENTENCES, cosine, vectors): similarity_table = "\n".join( f"{cosine(vectors[0], vectors[index]):.2f} {SENTENCES[index]}" for index in range(len(SENTENCES)) ) print(f"similarity to: {SENTENCES[0]}\n\n{similarity_table}") return @app.cell(hide_code=True) def _(SENTENCES, cosine, mo, vectors): mo.md(f""" Against *{SENTENCES[0]}*, the paraphrase scores **{cosine(vectors[0], vectors[1]):.2f}**, the opposite meaning scores **{cosine(vectors[0], vectors[2]):.2f}**, and the unrelated sentence scores **{cosine(vectors[0], vectors[3]):.2f}**. Read the first two numbers again. *{SENTENCES[2]}* means the opposite of *{SENTENCES[0]}* and it scores **{cosine(vectors[0], vectors[2]) - cosine(vectors[0], vectors[1]):.2f}** higher than the sentence that means the same thing, because it shares three words out of four. Counting words measures overlap and has no access to meaning. ๐Ÿ‘‰ **Try it:** work out by hand why the last sentence scores what it does, then check it. Then write two sentences that mean the same thing and score 0. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### What an embedding model changes Here are two sentences that mean the same thing with no word in common. """) return @app.cell def _(cosine, to_vector, vectors): delayed = to_vector("delivery was delayed") delayed_similarity = 0.0 if not any(delayed) else cosine(vectors[0], delayed) delayed, delayed_similarity return (delayed_similarity,) @app.cell(hide_code=True) def _(delayed_similarity, mo): mo.md(f""" Their similarity is **{delayed_similarity:.2f}**, because the vocabulary above has no place for words that never appeared. That is the ceiling on counting. An embedding model produces the vector instead of counting, and it is trained so that *delivery was delayed* lands near *the order shipped late* without sharing a word. The vector is longer, a few hundred to a few thousand numbers, and none of the positions means anything you can name. **What does not change is everything after that.** The vectors go into the same `cosine` function above, unmodified. Search over documents, clustering, and *find me ones like this* are all this measure applied to better vectors. **Where the vectors come from is a separate question.** The Claude API has no embeddings endpoint, so an embedding comes from a different provider's API or from a model you run locally. That is a decision with a cost and a dependency attached, which is one reason nothing in this course requires it. ๐Ÿ‘‰ **Try it:** ask your agent which local Python libraries produce sentence embeddings with no API key, and what one of them would add to your project's install size. Then decide whether you want it. โœ” **Check your understanding** 1. Two sentences share three words out of four and mean opposite things. What does counting words give you, and why? 2. Cosine similarity ignores the length of a vector. What problem does that solve? 3. You swap counted vectors for a real embedding model. Which part of the code changes? 4. Give a business question you could answer with document similarity, and one you could not. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## Chapter challenge **An agent that looks something up.** Nothing here is due. It exists because every section above teaches one part of a round trip, and a working agent needs all of them at once. **Build an agent with one tool, and let the model decide when to call it.** The tool calls a public API that needs no key. Kraken's ticker from section 31 works; so does anything else you can reach without signing up. Done looks like a program that: - reads a key from `.env` and fails loudly when it is missing - defines one tool, with a description you wrote and a schema that matches your function - runs the loop until `stop_reason` is `end_turn`, or until a step limit you set - prints the total input and output tokens for the whole run, and the cost - answers a question that cannot be answered without the tool **Four decisions are yours, and the notebook cannot make them.** 1. **What is the step limit, and what happens when you hit it?** Stopping silently and stopping with a message are different programs. 2. **What does your tool return when the lookup fails?** A `None`, a string saying so, or a `tool_result` with `is_error` set. Each one makes the model behave differently. 3. **What goes in the system prompt?** Write it, then delete it and run again, and describe what changed. 4. **How do you know the answer is right?** The model will produce a number. Name the check that would catch it being wrong. Hand in one file plus a markdown cell naming the four decisions and what the alternative would have been. **Going further.** Give it a second tool and a question that needs both, then read the conversation to see what order it chose. Then add a tool that can fail and make it fail on purpose. Then run the same question five times and count how many times the sequence of tool calls was identical. Then send the whole thing to somebody outside this course, watch them use it, and write down what they asked for that it could not do. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## ๐Ÿค– Ask your agent Two different uses, and the second one is the one this course grades. ### Ask it about what this chapter left out Every one of these is real and had no room above. Run whatever comes back. - Ask what **streaming** is, why a long answer arrives in pieces, and what `client.messages.stream(...)` gives you that `create` does not. - Ask what **prompt caching** is, and why resending the same 5,000-token schema on every turn can cost a tenth as much when you mark it. Section 34's arithmetic is the reason to care. - Ask what **structured outputs** are, and how you get JSON back that matches a schema you specified instead of JSON you have to hope about. - Ask what a **429** is, what the library already does about it on your behalf, and how many times it retries before giving up. - Ask what `stop_reason` of `refusal` means, what is in `stop_details`, and why retrying the identical request is the wrong response to it. ### Watch what it decides without telling you Give it this, exactly: > *"Write Python that calls a model and summarizes a customer complaint."* Then read what came back and answer four questions. 1. **Which model id did it use, and where did that come from?** You did not name one. 2. **What is `max_tokens`, and how did it pick that number?** 3. **Where did it put the key?** Look for a literal string, an `os.getenv`, or nothing at all. 4. **What does it do when the reply is not a single text block, or when the request fails?** Read the code, and ignore any comment claiming it handles errors. You asked for one thing and got four or five decisions. That gap is the subject of this course, and it does not go away as the models get better. It gets quieter. ๐Ÿ‘‰ **Try it:** send the same sentence twice, in two separate conversations. Compare the two `max_tokens` values. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## Chapter D checklist - [ ] I can send an HTTP request and read the status, the headers and the body - [ ] I check the shape of a response before I index into it - [ ] I can tell a `KeyError` from a parsing failure, and explain what each one means - [ ] I check the type of each value in a response, because a price can arrive as text - [ ] When a library makes the request for me, I check for an empty result before I index - [ ] I can set up `.env`, `.gitignore` and `.env.example`, and explain what each is for - [ ] I read a key with `os.getenv` and fail loudly when it is missing - [ ] I can name four places a key must never go, and what to do when one leaks - [ ] I can name the four parts of a request to a model - [ ] I know `system` is a top-level parameter, and what a naive port does to it - [ ] I read a reply by looping over its blocks and checking the type - [ ] I can explain why the model has no memory and where the memory actually is - [ ] I know which line makes a chat loop remember, and what breaks when it is missing - [ ] I run a prompt more than once before I believe a number that came out of it - [ ] I can write a tool definition whose schema matches my function - [ ] I can run the tool the model asked for and send the result back correctly - [ ] I can write the agent loop and state what its stop condition is - [ ] I put a step limit on any loop that calls a model - [ ] I can read `usage` off a reply and turn it into a cost - [ ] I can explain why turn 40 of a conversation costs more than turn 1 - [ ] I can explain what an embedding is for without doing any of the math This is the last chapter of the Python handbook. The SQL handbook covers the database side, and the two meet whenever a question needs both. """) return if __name__ == "__main__": app.run()