# /// script # requires-python = ">=3.12,<3.15" # dependencies = [ # "marimo", # ] # /// """Python Handbook, Chapter B: Collections. """ import marimo __generated_with = "0.24.0" app = marimo.App(width="medium", sql_output="polars") @app.cell def _(): import marimo as mo return (mo,) @app.cell(hide_code=True) def _(mo): mo.md(r""" # Collections Chapter B of the Python Handbook. Nine sections, each one concept. Chapter A held one value in one name. This chapter holds many, and the shape you choose decides what questions are easy to ask. Every number below comes from orders `10248` to `10252` of the course warehouse, which is the same table SQL queries later in the term. Small enough to check by hand, and real. You can read this straight through, or open it when you need one thing. Every section ends with three questions that have no answers written down, because those are the ones asked out loud in class. To run the code yourself, download the `.py` into your project and open it with `uv run marimo edit`. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## 11. Lists A list holds several values in one name, in the order you put them there. It is the first collection you reach for and the one most analytics code is built out of. ๐Ÿ“– **Used in:** Getting everything running ยท What one row means """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Square brackets, commas, and `len()` The freight charged on orders 10248 to 10252. """) return @app.cell def _(): freight_charges = [16.75, 22.25, 25.00, 20.25, 36.25] freight_charges, len(freight_charges) return (freight_charges,) @app.cell(hide_code=True) def _(mo): mo.md(r""" ๐Ÿ‘‰ **Try it:** add a sixth charge to the literal and run it again. Then run `sum(freight_charges)` and `max(freight_charges)` in a cell of your own. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Positions and slices `freight_charges[0]` is the first one, and the number in the brackets is its **index**. A negative index counts back from the end, and a slice reads as **from here up to but not including there**. This is the same counting a string uses in ยง5. """) return @app.cell def _(freight_charges): freight_charges[0], freight_charges[-1], freight_charges[1:3] return @app.cell(hide_code=True) def _(mo): mo.md(r""" ๐Ÿ‘‰ **Try it:** get the last two charges two different ways, one with a negative number and one without. **The trap.** There are five charges and the last position is 4. Asking for position 5 raises: ```python freight_charges[5] ``` ``` IndexError: list index out of range ``` """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### A list can be changed in place This is the difference between a list and everything in Chapter A. A string method hands back a new string; a list method changes the list you already have, which is what it means to call a list **mutable**. `.append()` adds to the end, and assigning to a position replaces what was there. """) return @app.cell def _(): ship_countries = ["France", "Germny", "Brazil", "France"] ship_countries[1] = "Germany" ship_countries.append("Belgium") ship_countries, len(ship_countries) return (ship_countries,) @app.cell(hide_code=True) def _(mo): mo.md(r""" Five orders shipped to four countries, because France appears twice. Hold on to that; it is what makes ยง13 and ยง17 worth reading. ๐Ÿ‘‰ **Try it:** `.insert(0, "Sweden")` puts a value at a position and pushes the rest along. `.remove("Brazil")` deletes the first match. `.pop()` takes the last one off and hands it back to you. Run all three and print the list after each one. **`.append()` adds one item, whatever you hand it.** Hand it a list of two and you get one item that happens to be a list, and `len()` goes up by one rather than two. `.extend()` is the one that adds them separately. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### `sorted()` and `.sort()` Two ways to sort, and they do different things to the list you already have. """) return @app.cell def _(freight_charges): freight_sorted = sorted(freight_charges) sort_demo = [16.75, 22.25, 25.00, 20.25, 36.25] sort_return = sort_demo.sort() freight_sorted, sort_return, sort_demo, freight_charges return (sort_return,) @app.cell(hide_code=True) def _(mo): mo.md(r""" Five values, in order. `sorted()` produced a sorted copy and left `freight_charges` in its original order. `.sort()` put `sort_demo` in order and handed back `None`. **The trap, and it is the most common one in this section.** `.sort()` returns nothing, so a line that looks like the obvious way to write it throws your data away: ```python charges = [16.75, 22.25, 25.00] charges = charges.sort() # charges is now None ``` Nothing raises on that line. The failure arrives later, wherever `charges` gets used. """) return @app.cell def _(sort_return): sort_return is None, type(sort_return) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ๐Ÿ‘‰ **Try it:** run `sort_return[0]` in a cell of your own and read the error. That is the message you would actually meet, several cells away from the line that caused it. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Names that share a list `=` puts a value into a name, and for a list that value is the list itself. Two names can point at one list, each an **alias** for the other, and a change through either name is visible through both. `.copy()` is how you get a second list. """) return @app.cell def _(): original_freight = [16.75, 22.25, 25.00] same_list = original_freight freight_copy = original_freight.copy() same_list[0] = 99.99 original_freight, freight_copy, same_list is original_freight, freight_copy is original_freight return @app.cell(hide_code=True) def _(mo): mo.md(r""" One assignment, through `same_list`, and `original_freight` changed with it. `freight_copy` did not. `is` and `==` answer different questions. `==` asks whether two lists hold equal values. `is` asks whether there is only one list, which is a question about **identity**. Two separate lists holding the same three numbers are `==` and are not `is`. ๐Ÿ‘‰ **Try it:** add `original_freight == freight_copy` to that cell before the assignment line, and again after it. The answer changes and the `is` answers do not. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### In Excel terms Excel's **Data > Sort** rewrites the sheet. There is no version of it that hands you a sorted copy and leaves the sheet alone, so the habit is to expect a sort to *be* the answer. `sorted()` is the version Excel does not have, and `.sort()` is the one it does. Writing `x = lst.sort()` is that Excel habit reaching for the wrong one. ### โœ” Check your understanding 1. `sorted(charges)` and `charges.sort()` both put five numbers in order. Describe what each one leaves behind afterward. 2. You write `charges = charges.sort()` and three cells later something fails. What does `charges` hold, and why did the failure not happen on the line you wrote? 3. `b = a` and `b = a.copy()` both give you a name `b` holding three numbers. Give a situation where the difference between them changes an answer. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## 12. Iterating over a list with `for` A `for` loop runs the same block of code once for each item in a list, which is what it means to **iterate** over the list, and anything you can loop over this way is an **iterable**. This is where code stops describing one thing and starts describing a procedure that scales to a million rows without changing. ๐Ÿ“– **Used in:** What one row means """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### The shape of a loop The line ends in a colon and the block underneath is indented, the same shape as an `if` in ยง4. The name between `for` and `in` is yours to choose, and it holds one item at a time. """) return @app.cell def _(freight_charges): for charge in freight_charges: print(charge) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ๐Ÿ‘‰ **Try it:** rename `charge` to `c` and run it again. Nothing changes except how readable the loop is. Then, with whichever name you settled on, print it formatted: `f"{charge:.2f}"`, so every line has two decimals. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Building a total A running total needs a name that exists before the loop starts. Each pass adds one item to it. """) return @app.cell def _(freight_charges): freight_running_total = 0.0 for each_charge in freight_charges: freight_running_total = freight_running_total + each_charge freight_running_total return @app.cell(hide_code=True) def _(mo): mo.md(r""" **The trap.** Move `freight_running_total = 0.0` inside the loop and it is reset on every pass: ```python for each_charge in freight_charges: freight_running_total = 0.0 freight_running_total = freight_running_total + each_charge # freight_running_total is 36.25, the last charge, not 120.5 ``` One line moves and the answer stops being a total. Nothing raises, and 36.25 is a plausible freight number. ๐Ÿ‘‰ **Try it:** run both versions and say the two answers out loud. **Rename the total in your copy**, to `broken_running_total`, because a name is defined in exactly one cell here and a second `freight_running_total` is a multiple-definitions error rather than the lesson. Then predict what happens if you move the `0.0` line inside the loop but put the total on the line *after* the loop ends. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### `enumerate` when you need the position too Most loops need the item and nothing else. When you also need to know which one it is, `enumerate` hands you both, and `start=1` counts the way a person would. """) return @app.cell def _(freight_charges): for position, listed_charge in enumerate(freight_charges, start=1): print(f"{position}. {listed_charge:>6.2f}") return @app.cell(hide_code=True) def _(mo): mo.md(r""" ๐Ÿ‘‰ **Try it:** drop `start=1` and run it again. Then print the order number instead of the position, using `10247 + position`. **A pointer forward.** A loop that only builds a new list from an old one can be written on one line, and agent-written Python does this constantly. That one line is called a comprehension, it is the same loop, and ยง18 is where you read one. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### `while` repeats until its condition is `False` A `for` loop makes one pass per item. `while` is for the case where the number of passes is unknown: it checks a condition before every pass and stops the first time the condition is `False`. Here it counts the years until an investment growing at 7% doubles. """) return @app.cell def _(): growth_rate = 0.07 investment_value = 10_000.00 years_to_double = 0 while investment_value < 20_000: investment_value = investment_value * (1 + growth_rate) years_to_double = years_to_double + 1 years_to_double return @app.cell(hide_code=True) def _(mo): mo.md(r""" Something inside the loop has to move the condition toward `False`. At a `growth_rate` of `0` the value never changes and the cell runs until you press stop. Code an agent writes often uses `while True:` with a `break` inside, and `break` ends the loop on the spot. The agent loop in [ยง37](https://oim6301.github.io/handbooks/python/04-agents-and-apis/#37-the-agent-loop) has this shape. Read it as "repeat until", then find the line that ends it. When each pass costs money, as an API call does, give the loop a ceiling instead: ```python for attempt in range(5): # at most five passes ... if done: break ``` ### โœ” Check your understanding 1. What does the name between `for` and `in` hold on the third pass of a loop over five charges? 2. A colleague's total comes out as the last value in the list every time. Where is the bug, and what is the symptom telling you? 3. When would you reach for `enumerate` instead of a plain `for`? 4. A `while` loop has been running for a minute. Which line do you read first, and what are you looking for in it? """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## 13. Filtering and the accumulator An `if` inside a loop keeps some items and skips the rest. An accumulator carries a running answer across the passes. Put the two together and you have written what SQL calls `WHERE` and `GROUP BY`, by hand, before you have met either one. ๐Ÿ“– **Used in:** What one row means """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### `if` inside the loop keeps some items Start with an empty list, test each item, and append the ones that pass. """) return @app.cell def _(freight_charges): heavy_freight = [] for candidate in freight_charges: if candidate > 22: heavy_freight.append(candidate) heavy_freight, len(heavy_freight) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Three of the five charges are above 22. ๐Ÿ‘‰ **Try it:** change `> 22` to `>= 22.25` and predict the count before you run it. Then write the filter that keeps the charges *below* the average of 24.10. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### A list of records A list of numbers can be totalled. Answering *which country* costs the most needs the country attached to the charge, which means each item carries several named fields. The shape below is a **dictionary**, one per order, and ยง15 gives dictionaries their own section. Here it is only the thing being looped over: `order["ship_country"]` reads one field out of one order. """) return @app.cell def _(): orders = [ {"order_id": 10248, "ship_country": "France", "freight": 16.75, "days_to_ship": 12}, {"order_id": 10249, "ship_country": "Germany", "freight": 22.25, "days_to_ship": 5}, {"order_id": 10250, "ship_country": "Brazil", "freight": 25.00, "days_to_ship": 4}, {"order_id": 10251, "ship_country": "France", "freight": 20.25, "days_to_ship": 7}, {"order_id": 10252, "ship_country": "Belgium", "freight": 36.25, "days_to_ship": 2}, ] len(orders), orders[0]["ship_country"], orders[0]["freight"] return (orders,) @app.cell(hide_code=True) def _(mo): mo.md(r""" ๐Ÿ‘‰ **Try it:** print the country and the freight of the last order, using a negative position. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### A rate from accumulators A count and a total are both accumulators. Keep them side by side and you can report an average and a share from one pass over the data. """) return @app.cell def _(orders): slow_count = 0 days_total = 0 for timed_order in orders: days_total = days_total + timed_order["days_to_ship"] if timed_order["days_to_ship"] >= 7: slow_count = slow_count + 1 days_total, slow_count, days_total / len(orders), slow_count / len(orders) return @app.cell(hide_code=True) def _(mo): mo.md(r""" The five orders took 30 days between them, so a mean of 6.0, and two of them took a week or longer, which is 0.4 of them. **The trap.** `days_total / len(orders)` divides by the number of orders you started with. If the loop had skipped some of them, that denominator is wrong and the average is quietly too low. Count what you actually included. ๐Ÿ‘‰ **Try it:** add a second counter that only increments inside the `if`, and divide the total days of the slow orders by it. Explain what that number means and how it differs from 6.0. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### `continue` skips the rest of this pass An early `continue` reads as a filter at the top of the loop, which keeps the interesting work out of an indented block. """) return @app.cell def _(orders): france_freight = 0.0 for scanned_order in orders: if scanned_order["ship_country"] != "France": continue france_freight = france_freight + scanned_order["freight"] france_freight return @app.cell(hide_code=True) def _(mo): mo.md(r""" Order `10248` cost 16.75 and order `10251` cost 20.25, so France comes to 37.0. ๐Ÿ‘‰ **Try it:** rewrite that loop with an `if` around the addition and no `continue`. Both give 37.0. Decide which one you would rather read at 6pm. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Aggregating by key The loop above answers one country at a time. Doing every country at once needs a place to keep a running total *per country*, and that place is a dictionary: the country is the key, the running total is the value. Three steps, once per order. Read the key out of the record. Make sure it has a slot. Add this order's freight to whatever the slot already holds. """) return @app.cell def _(orders): freight_by_country = {} for grouped_order in orders: country = grouped_order["ship_country"] if country not in freight_by_country: freight_by_country[country] = 0.0 freight_by_country[country] = freight_by_country[country] + grouped_order["freight"] freight_by_country return @app.cell(hide_code=True) def _(mo): mo.md(r""" Four keys out of five orders, and France holds 37.0 because two orders landed in it. The SQL that returns this same answer is one statement, and you will meet it in the SQL handbook: ```sql SELECT "ShipCountry", SUM("Freight") FROM orders WHERE "OrderID" BETWEEN 10248 AND 10252 GROUP BY "ShipCountry"; ``` The loop above is what the database does. Knowing that is why `GROUP BY` will not feel like magic. ๐Ÿ‘‰ **Try it:** change the value being added from `grouped_order["freight"]` to `1` and you have a count per country instead of a sum. Predict all four numbers before you run it. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### In Excel terms `SUMIF` and a pivot table both do this, and both hide the loop. The habit that misleads is case: Excel's `SUMIF` criteria are case-insensitive, so `france` and `France` land in one bucket without you asking. A dictionary key is case-sensitive, so the same two spellings become two entries, each with part of the answer. Nothing warns you, and both numbers look reasonable. ### โœ” Check your understanding 1. Five orders go in and the dictionary comes out with four keys. Where did the fifth order go? 2. Your average days-to-ship is computed as `total / len(orders)` inside a loop that skips cancelled orders. What is wrong with the answer, and which direction is it wrong in? 3. A country is spelled `"France"` in four rows and `"france"` in one. What does the aggregate look like, and how would you notice? """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## 14. Tuples A tuple holds several values in order, like a list, and is **immutable**, so it cannot be changed after it is made. The second half is the whole point, and it buys you two things a list cannot offer. ๐Ÿ“– **Used in:** Choosing a data structure """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Tuples One line of order `10248`: a product, a quantity, a unit price. """) return @app.cell def _(): first_line = ("Queso Cabrales", 12, 14.00) first_line, len(first_line), first_line[0] return (first_line,) @app.cell(hide_code=True) def _(mo): mo.md(r""" **The trap.** Everything a list can do to itself, a tuple refuses: ```python first_line[1] = 20 ``` ``` TypeError: 'tuple' object does not support item assignment ``` ๐Ÿ‘‰ **Try it:** run `first_line.append(9.80)` and read which of the two things it tells you. The message names the missing method, and the reason it is missing is the paragraph above. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Unpacking gives each part its own name Three names on the left, one tuple on the right, and the parts line up by position. This is **unpacking**, and it is also how a function returns several values at once, because `return total, count` hands back one tuple. """) return @app.cell def _(first_line): product_name, line_quantity, line_price = first_line product_name, line_quantity * line_price return @app.cell(hide_code=True) def _(mo): mo.md(r""" 12 at 14.00 comes to 168.0, which is the number you would check by hand. ๐Ÿ‘‰ **Try it:** unpack it into two names instead of three and read the error. Then write a function that takes a quantity and a price and returns both the line total and the quantity, as a tuple. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### When a tuple is the right answer A list is the default. Three situations turn that around, and all three are triggers you can check, and not qualities to weigh up. 1. **The number of parts is fixed and each position means something.** A line is always a product, a quantity and a price. Appending a fourth thing to it would be a bug, and a tuple makes that bug impossible. 2. **Something else is going to hold on to it.** A tuple you hand to another function cannot come back changed, which removes a class of surprise from code you did not write. 3. **You need it as a dictionary key.** This one is not a preference. A list cannot be a key and a tuple can. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### A tuple as a dictionary key ยง13 grouped freight by one field. Grouping by two fields at once needs a key made of both, and a tuple is what that key is. """) return @app.cell def _(): freight_by_country_year = { ("France", 2016): 37.00, ("Germany", 2016): 22.25, ("Brazil", 2016): 25.00, ("Belgium", 2016): 36.25, } freight_by_country_year[("France", 2016)], len(freight_by_country_year) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ๐Ÿ‘‰ **Try it:** try the same dictionary with `["France", 2016]` as a key instead. The error names the property a key has to have. That property is being **hashable**, and a list does not have it. ### โœ” Check your understanding 1. A tuple and a list both hold three values in order. Name one thing you can do to a list that you cannot do to a tuple, and explain why that is sometimes what you want. 2. You need to total revenue by region and channel together. What is the key, and what type is it? 3. A function returns `return total, count`. What type did the caller receive, and how do they get the two numbers apart? """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## 15. Dictionaries A dictionary stores values by name instead of by position. `order["freight"]` says what it reads; `order[2]` does not. Everything a row of data is, in Python, is a dictionary. ๐Ÿ“– **Used in:** Choosing a data structure """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Keys, values, and access by key Braces, and each entry is `key: value`. Order `10248` in full. """) return @app.cell def _(): order_10248 = { "order_id": 10248, "ship_country": "France", "freight": 16.75, "ordered": "2016-07-04", "shipped": "2016-07-16", } order_10248["ship_country"], order_10248["freight"], len(order_10248) return (order_10248,) @app.cell(hide_code=True) def _(mo): mo.md(r""" Five fields, and `len()` counts the keys. It says nothing about the values. ๐Ÿ‘‰ **Try it:** read the two dates out and compare them as strings. Then state what ยง7 of Chapter A would make you do before trusting that comparison. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### A missing key and `.get()` Asking for a key that is not there stops the program: ```python order_10248["discount"] ``` ``` KeyError: 'discount' ``` `.get()` reads the same key and hands back a default when it is missing. It is the one-line version of the `if country not in freight_by_country` line ยง13 wrote out. """) return @app.cell def _(order_10248): discount_10248 = order_10248.get("discount", 0.0) country_10248 = order_10248.get("ship_country", "unknown") discount_10248, country_10248, "discount" in order_10248 return @app.cell(hide_code=True) def _(mo): mo.md(r""" **The trap.** A default is a decision. `order.get("discount", 0.0)` says a missing discount means no discount, and that is a business claim about your data. Where a missing field means *nobody filled this in*, `0.0` is a number you invented and it will be averaged with the real ones. ๐Ÿ‘‰ **Try it:** call `.get("discount")` with no default and print what comes back. Then add `1` to it and read the error. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Adding and changing keys Assigning to a key that exists replaces its value. Assigning to one that does not adds it. Dictionaries change in place, the same way lists do, so a copy is a real decision. """) return @app.cell def _(order_10248): order_10248_edited = dict(order_10248) order_10248_edited["freight"] = 18.00 order_10248_edited["carrier"] = "Speedy Express" len(order_10248), len(order_10248_edited), "carrier" in order_10248 return @app.cell(hide_code=True) def _(mo): mo.md(r""" Five keys became six, and the original still has five because `dict(...)` made a second dictionary. ๐Ÿ‘‰ **Try it:** drop the `dict(...)` and assign `order_10248_edited = order_10248` instead. Then check `len(order_10248)` again. ยง11 already told you what happens. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### `.items()` loops over both halves A `for` over a dictionary alone gives you the keys. `.items()` gives you the key and the value together, which is what you want nine times out of ten. """) return @app.cell def _(order_10248): for field_name, field_value in order_10248.items(): print(f"{field_name:<14} {field_value}") return @app.cell(hide_code=True) def _(mo): mo.md(r""" ๐Ÿ‘‰ **Try it:** print the freight totals from ยง13 the same way, sorted by the country name. `sorted(freight_by_country.items())` is the whole change. ### โœ” Check your understanding 1. `len()` on a dictionary of five fields returns 5. Five of what? 2. `order.get("discount", 0)` never raises. Give a case where that is exactly the problem. 3. You loop `for x in some_dict:` and print `x`. What are you printing, and what would you write to get the values instead? """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## 16. A list of dicts is a table One dictionary is one row. A list of them is a table. That sentence is the spine of this chapter and of the whole course, because it is the same object SQL queries and the same object polars loads, and every one of the three calls it something different. ยง13 already looped over five orders shaped this way. This section is what that shape is. ๐Ÿ“– **Used in:** What one row means ยท Choosing a data structure """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### A dict as a row The three lines of order `10248`, exactly as the warehouse holds them. """) return @app.cell def _(): order_lines = [ {"product": "Queso Cabrales", "category": "Dairy Products", "quantity": 12, "unit_price": 14.00}, {"product": "Mozzarella di Giovanni", "category": "Dairy Products", "quantity": 5, "unit_price": 34.80}, {"product": "Singaporean Hokkien Fried Mee", "category": "Grains/Cereals", "quantity": 10, "unit_price": 9.80}, ] len(order_lines), list(order_lines[0].keys()) return (order_lines,) @app.cell(hide_code=True) def _(mo): mo.md(r""" Three rows and four columns. The keys of the first row *are* the column headers, which is why `list(order_lines[0].keys())` is the closest thing this shape has to a header row. **The trap.** Nothing forces the rows to agree. A fourth row missing `unit_price` is a legal list of dicts and an illegal table, and you find out when something reads that key. ๐Ÿ‘‰ **Try it:** add a fourth line with only `product` and `quantity` in it. Then run `list(order_lines[3].keys())` and compare it to row 0. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Building the table row by row A literal is fine when you type the data yourself. Everything else arrives one row at a time, so the shape you will actually write is an empty list and an `append` per row. """) return @app.cell def _(): built_lines = [] built_lines.append({"product": "Queso Cabrales", "quantity": 12, "unit_price": 14.00}) built_lines.append({"product": "Mozzarella di Giovanni", "quantity": 5, "unit_price": 34.80}) built_lines.append({"product": "Singaporean Hokkien Fried Mee", "quantity": 10, "unit_price": 9.80}) len(built_lines), built_lines[2]["product"] return @app.cell(hide_code=True) def _(mo): mo.md(r""" ๐Ÿ‘‰ **Try it:** wrap those three appends in a loop over a list of three tuples, so the dictionary is built inside the loop. That is what chapter C's CSV reader does to every line of a file. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Totaling a column A loop over the rows, reading one key out of each. Order `10248` before freight is 440, and the freight on it is 16.75. """) return @app.cell def _(order_lines): subtotal = 0.0 for priced_line in order_lines: subtotal = subtotal + priced_line["quantity"] * priced_line["unit_price"] subtotal, f"${subtotal:,.2f}", f"${subtotal + 16.75:,.2f}" return @app.cell(hide_code=True) def _(mo): mo.md(r""" 440.0 as a number, `$440.00` as a sentence, and `$456.75` with freight added. The three line amounts are 168.0, 174.0 and 98.0, and they are worth adding up on paper once. ๐Ÿ‘‰ **Try it:** total the `quantity` column instead, then compute the average unit price. Judge whether that average is a number you would put in front of anybody. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Sorting the rows by a field `sorted()` on a list of dicts has no idea which field to order by, so you tell it. `key=` takes a function that is handed one row and returns the thing to sort on, and `lambda row: ...` is how that function gets written inline. `reverse=True` makes it descending. """) return @app.cell def _(order_lines): by_line_total = sorted(order_lines, key=lambda row: row["quantity"] * row["unit_price"], reverse=True) by_line_total[0]["product"], by_line_total[1]["product"], by_line_total[2]["product"] return @app.cell(hide_code=True) def _(mo): mo.md(r""" Mozzarella at 174.0 outranks Queso at 168.0, even though Queso is 12 units against 5. Sorting by `quantity` would have put them the other way round, and both orderings are correct answers to different questions. ```sql SELECT "ProductID", "Quantity" * "UnitPrice" AS line_total FROM "Order Details" WHERE "OrderID" = 10248 ORDER BY line_total DESC; ``` That is the same three rows in the same order, named by `ProductID` because the names live in another table. `ORDER BY ... DESC` is `key=` plus `reverse=True`, and `SELECT` is the key you read out of each row. ๐Ÿ‘‰ **Try it:** sort by `category` and then by `quantity` descending, in one call. A `key=` that returns a tuple is how, and ยง14 says why a tuple. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Adding a computed column without touching the original `{**row, "new_key": value}` makes a new dictionary holding everything the old one had plus one more field. Build a new list out of those and the source rows are untouched. """) return @app.cell def _(order_lines): lines_with_total = [] for source_line in order_lines: lines_with_total.append({**source_line, "line_total": source_line["quantity"] * source_line["unit_price"]}) len(lines_with_total[0]), len(order_lines[0]), lines_with_total[0]["line_total"] return @app.cell(hide_code=True) def _(mo): mo.md(r""" Five keys in the new row, four in the original, and the first line total is 168.0. ๐Ÿ‘‰ **Try it:** add a second computed column in the same pass, `"is_dairy"`, from the `category` field. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Copying the list does not copy the rows ยง11 said `.copy()` gives you a second list. It does. What it does not do is give you second dictionaries: both lists hold the *same* rows, so editing a row through either list changes it for both. That is a **shallow copy**, and `copy.deepcopy()` is the **deep copy** that goes all the way down. """) return @app.cell def _(): import copy two_lines = [ {"product": "Queso Cabrales", "quantity": 12}, {"product": "Mozzarella di Giovanni", "quantity": 5}, ] shallow_copy = two_lines.copy() deep_copy = copy.deepcopy(two_lines) shallow_copy[0]["quantity"] = 99 deep_copy[1]["quantity"] = 55 two_lines, deep_copy return @app.cell(hide_code=True) def _(mo): mo.md(r""" Read the two outputs side by side. The 99 written through `shallow_copy` is sitting in `two_lines`. The 55 written through `deep_copy` is not. **The trap, and it is the one that bites this shape hardest.** Cleaning a copy of your data is the reason anybody copies it, and `.copy()` on a list of dicts is the version of that which quietly edits the original. Build a new list of new dictionaries, the way the computed-column cell above does, or reach for `copy.deepcopy`. ๐Ÿ‘‰ **Try it:** add `shallow_copy[0] = {"product": "Something else", "quantity": 1}` to that cell and check `two_lines` again. Replacing a whole row behaves differently from editing one, and the difference is the point. ### โœ” Check your understanding 1. You are handed a list of 500 dictionaries. What is the fastest way to find out what the columns are, and what could that answer miss? 2. `sorted(rows)` raises on a list of dicts. What do you have to supply, and what is it for? 3. You copy a list of rows, clean the copy, and the original is clean too. What happened, and what would you have written instead? """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## 17. Sets A set holds each value once and keeps no order. It answers two questions faster and more clearly than a list does: what are the distinct values, and have I seen this one before. ๐Ÿ“– **Used in:** Choosing a data structure """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Unique values without positions Five orders shipped to four countries, because France appears twice. `set()` is how you express that in one line. """) return @app.cell def _(ship_countries): distinct_countries = set(ship_countries) len(ship_countries), len(distinct_countries), sorted(distinct_countries) return (distinct_countries,) @app.cell(hide_code=True) def _(mo): mo.md(r""" The `sorted()` is there for a reason. A set prints in whatever order it happens to hold its values, and that order can differ between runs, so anything you show a person goes through `sorted()` first. **The trap.** A set has values and no positions: ```python distinct_countries[0] ``` ``` TypeError: 'set' object is not subscriptable ``` ๐Ÿ‘‰ **Try it:** build a set from the freight charges and count it. Then add a duplicate charge to the list and count both again. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### `in` is what a set is fast at `in` works on a list too. On a set it stays fast no matter how many values are in it, which is why a membership test over a large reference list belongs in a set. """) return @app.cell def _(distinct_countries): "France" in distinct_countries, "Spain" in distinct_countries return @app.cell(hide_code=True) def _(mo): mo.md(r""" ๐Ÿ‘‰ **Try it:** put the four country names in a list and run the same two tests. The answers match, and for four values the speed does too. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Comparing sets `&` keeps what is in both, `-` keeps what is only in the first, and `|` keeps everything from either. The countries of the first three orders against the last three. """) return @app.cell def _(): first_three_countries = {"France", "Germany", "Brazil"} last_three_countries = {"Brazil", "France", "Belgium"} ( sorted(first_three_countries & last_three_countries), sorted(first_three_countries - last_three_countries), sorted(first_three_countries | last_three_countries), ) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Brazil and France are in both windows, Germany is only in the first, and four countries appear across the week. ๐Ÿ‘‰ **Try it:** run `last_three_countries - first_three_countries` and confirm it gives a different answer from the one above. Subtraction has a direction and intersection does not. ### โœ” Check your understanding 1. A list of 10,000 order countries becomes a set of 21. What did you learn, and what did you lose? 2. You need to know which products were ordered in July and not in August. Which set operation, and in which direction? 3. Why does this section put `sorted()` around every set it displays? """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## 18. Comprehensions > **Advanced** A comprehension builds a list in one line out of another list. You are here to **read** them: an agent asked for anything in this chapter will write one, and the code you are checking will be full of them long before you choose to write your own. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Comprehensions as loops Transform every row. Keep some rows. Do both. The `for` is in the middle of the line, the thing being collected is at the front, and the condition is at the end. """) return @app.cell def _(order_lines): product_names = [line["product"] for line in order_lines] dairy_lines = [line for line in order_lines if line["category"] == "Dairy Products"] dairy_names = [line["product"] for line in order_lines if line["category"] == "Dairy Products"] product_names, len(dairy_lines), dairy_names return @app.cell(hide_code=True) def _(mo): mo.md(r""" Three products, two of them dairy. The second form keeps whole rows and the third keeps one field out of the rows it kept, which is the difference between `SELECT *` and `SELECT ProductName`. **How to read one.** Find the `for` first and read the middle. That tells you what is being looped over. Then read the end, which is the filter. Then read the front, which is what comes out. Front, middle, end is the order it is written in and the wrong order to read it in. ๐Ÿ‘‰ **Try it:** write the first line as a four-line `for` loop with an `append`. Then decide which of the two you would rather find in code somebody else wrote. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### A comprehension builds a new list and changes nothing Whatever it was reading is exactly as it was afterward. That is worth knowing when you are auditing agent output, because it means a comprehension is never the line that edited your data. """) return @app.cell def _(order_lines): quantity_by_product = {line["product"]: line["quantity"] for line in order_lines} quantity_by_product, len(order_lines) return @app.cell(hide_code=True) def _(mo): mo.md(r""" Braces instead of brackets, and a `key: value` in front, and you get a dictionary instead of a list. Indexing a table by one of its fields is what this form is usually doing. **The trap.** A dictionary keeps one value per key. If two rows share a product name, the second one silently replaces the first and the count drops with no warning. ๐Ÿ‘‰ **Try it:** add a fourth line to `order_lines` with `"product": "Queso Cabrales"` and a different quantity. Then check `len(quantity_by_product)` against `len(order_lines)`. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### When to write the loop instead A comprehension that carries two conditions, a nested `for` and a calculation is a sentence with no punctuation. The loop version is longer, and the reason to prefer it is that you can put a `print` in the middle of a loop and cannot put one in the middle of a comprehension. That is also the honest test for agent-written code. If you cannot explain out loud what a comprehension collects, ask for it as a loop and read that. ๐Ÿ‘‰ **Try it:** ask an agent to rewrite `dairy_names` as a plain loop, and then to rewrite one of your own loops as a comprehension. Run both and compare the answers, not the prose. ### โœ” Check your understanding 1. `[o["freight"] for o in orders if o["days_to_ship"] > 7]` produces what, and how many items does it have? 2. Which part of a comprehension do you read first, and why is it not the first part written? 3. A dict comprehension over 500 rows gives 480 keys. What happened to the other 20? """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## 19. Choosing a data structure Five shapes, which most books call **data structures**, have now gone past: list, tuple, dictionary, list of dicts, set. Choosing between them is a procedure with four questions, and you answer them in order. ๐Ÿ“– **Used in:** Choosing a data structure ยท Getting data from an API ยท Designing your database """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### Questions for choosing a structure 1. **Do you need each value only once, such as the distinct countries, or every customer counted once?** Then it is a **set**, and nothing else in the list applies. 2. **Is this one thing with named parts?** A single order, a single customer, a single configuration. Then it is a **dictionary**, with the part names as keys. If the parts are fixed and must never change, and especially if this is going to be a dictionary key, it is a **tuple** instead. 3. **Is it many of the same thing?** Then it is a **list**. The next question decides what is inside it. 4. **Does each of those things have named parts?** Yes gives you a **list of dictionaries**, which is a table and is the right answer most of the time in this course. No gives you a plain list of values, which is right when there is genuinely only one number per item. The order matters, because question 4 is the one people skip. A list of values is easy to type and stops being enough the moment somebody asks *which country*. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### The shape that looks fine and is not Two lists side by side, one holding countries and one holding freight, with position 0 of each belonging to the same order. It works, right up until something reorders one of them. """) return @app.cell def _(): parallel_countries = ["France", "Germany", "Brazil", "France", "Belgium"] parallel_freight = [16.75, 22.25, 25.00, 20.25, 36.25] before_sort = list(zip(parallel_countries, parallel_freight)) parallel_freight.sort() after_sort = list(zip(parallel_countries, parallel_freight)) before_sort, after_sort return @app.cell(hide_code=True) def _(mo): mo.md(r""" `zip` pairs the two lists by position, which is the pairing the whole arrangement depends on. Read the two outputs against each other. Before the sort, Germany owes 22.25 and Brazil owes 25.0. After it, Germany owes 20.25 and Brazil owes 22.25. Three of the five rows are now wrong, every number is still a real freight charge, and nothing raised. **The two that survived are the reason this hides.** France keeps 16.75 and Belgium keeps 36.25, because those are the smallest and largest charges and the sort left them where they were. Spot-check the first row and the last row and the data looks fine. A list of dictionaries cannot do this to you, because the country and the charge are in the same row and sorting moves them together. That is most of the argument for ยง16. ๐Ÿ‘‰ **Try it:** rebuild those five orders as a list of dicts, sort it by freight, and confirm the countries follow their charges. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" ### The procedure on a real question *"Which country cost us the most in freight last week?"* Question 1: no, you are not testing membership. Question 2: no, this is five orders and not one. Question 3: yes, many of the same thing, so a list. Question 4: each order has an id, a country, a freight charge and a shipping time, so a **list of dictionaries**. Then the answer itself is a country and a number per country, which is one thing with named parts repeated, so it comes out as a **dictionary** keyed by country. That is exactly what ยง13 built. A shape is a bet about the questions you will be asked next. Getting it wrong is recoverable and rewriting the analysis around it is what costs the afternoon. ๐Ÿ‘‰ **Try it:** run the same four questions over *"how many distinct products did we sell in July"* and over *"what is our current exchange rate"*. Two questions, two different shapes, and neither is a list of dicts. ### โœ” Check your understanding 1. You have five orders and one freight charge each. Name two shapes that hold that, and explain what the second one lets you ask that the first does not. 2. Somebody hands you two lists that line up by position. What is the first question you ask them? 3. Question 4 in the procedure asks whether each item has named parts. Give an example from your own work where the answer is no. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## Chapter challenge **The first week of orders.** Nothing here is due. It exists because the sections above each teach one thing and one real question needs most of them at once. | OrderID | ship country | freight | days to ship | |---|---|---|---| | 10248 | France | 16.75 | 12 | | 10249 | Germany | 22.25 | 5 | | 10250 | Brazil | 25.00 | 4 | | 10251 | France | 20.25 | 7 | | 10252 | Belgium | 36.25 | 2 | **Build these five orders as one collection and answer three questions from it.** - Which country cost the most in freight across the week? - Which order took longest to ship? - What share of the week's orders shipped in under a week? **Four decisions are yours, and the notebook cannot make them.** 1. What shape holds the five orders? Name it, and state one thing it makes easy and one thing it makes hard. 2. "Under a week" is how many days? Write the comparison you mean and state what happens to order `10251`. 3. France appears twice. Does the answer keep them as two orders or as one country? Both are defensible, so state which question you are answering. 4. The freight values carry two decimals. Where in your calculation do you round, and why there? **Done looks like** one cell producing three answers, plus a markdown cell naming the four decisions and what the alternative would have been. **Going further.** Give order `10248` its three line items, so one record carries a list of dictionaries inside it, and compute the order total with freight. Then make your code survive an order whose line items are missing entirely. Then add a sixth order with the country typed as `"france"` and decide what should happen, in code and not by editing the data. Then hand the notebook to somebody outside this course and write down the first question they ask. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## ๐Ÿค– Ask your agent Two different uses, and the second one is the one this course grades. ### Ask it about what this chapter left out Every one of these is real Python that had no room above. - `collections.Counter` does ยง13's counting loop in one line. Ask what it gives back, and whether that thing is a dictionary. - `dict.setdefault` and `collections.defaultdict` are two more ways to write the same accumulator. Ask which one an agent reaches for by default, and why. - You met `zip` once, in ยง19. Ask what it does when the two lists are different lengths, and whether it warns you. - `sorted` accepts a `key` that returns a tuple. Ask how to sort orders by country and then by freight descending inside each country. - `any()` and `all()` take a list of comparisons. Ask what each returns for an empty list, and predict it before you read the answer. Test whatever it gives you. Paste it into a cell and run it. ### Watch what it decides without telling you Give it this, exactly, with the five orders pasted underneath: > *"Write Python that summarizes these five orders by country."* Then read what came back and answer four questions. 1. **What shape did it choose for the input, and did it change the shape you gave it?** 2. **Did it sort the answer?** By what, and in which direction? You did not ask for a sort. 3. **What does it do with `"france"` in lowercase?** Read the code before you test it. 4. **Does it round, and to how many places?** Find the line. You asked for a summary and got four or five decisions. That gap is the subject of this course, and it does not go away as the models get better. It gets quieter. ๐Ÿ‘‰ **Try it:** send the same sentence twice, in two separate conversations. Compare the two shapes it chose. """) return @app.cell(hide_code=True) def _(mo): mo.md(r""" --- ## Chapter B checklist - [ ] I can build a list, count it, and take one item or a slice out of it - [ ] I know that a list changes in place and a string does not - [ ] I can state what `sorted()` leaves behind and what `.sort()` leaves behind - [ ] I know what `x = lst.sort()` puts in `x`, and where the failure shows up - [ ] I can tell two names for one list from two separate lists, and I can make a copy - [ ] I can write a `for` loop over a list and choose a readable name for the item - [ ] I can build a running total, and I know why the zero goes outside the loop - [ ] I can filter inside a loop and collect what passes - [ ] I can keep a count and a total in one pass and get an average out of them - [ ] I can total one field grouped by another, and I know that is what `GROUP BY` does - [ ] I can explain why a dictionary key is case-sensitive and a `SUMIF` criterion is not - [ ] I can make a tuple, unpack it, and name a case where a list would be wrong - [ ] I can use a tuple as a dictionary key and explain why a list cannot be one - [ ] I can read a value out of a dictionary by key and handle a key that is missing - [ ] I know that a `.get()` default is a claim about my data - [ ] I can loop over a dictionary's keys and values together - [ ] I can describe what a list of dictionaries is, and where the column headers live in it - [ ] I can build a table row by row with `append` - [ ] I can sort a table by a field with `key=` and `reverse=True` - [ ] I can add a computed column without changing the original rows - [ ] I know that copying a list of dictionaries shares the dictionaries - [ ] I can read a comprehension by finding its `for` first - [ ] I can run the four questions and choose a shape before I write any code **Chapter C is Data**: getting somebody else's file in, and what it does to all of this once the rows are real. """) return if __name__ == "__main__": app.run()