{ "cells": [ { "cell_type": "markdown", "id": "770f00ea", "metadata": {}, "source": [ "# CSCI 3151 Readiness Arcade — Expanded Pre-Course Gym\n", "\n", "This notebook is a **pre-3151 repair kit**. It is meant to make the first weeks of CSCI 3151 feel less mysterious by repairing prerequisite math, NumPy, plotting, and evidence habits.\n", "\n", "The workflow is:\n", "\n", "1. Try the low-stakes **Start Screen Diagnostic**.\n", "2. Complete each Arcade after watching its video.\n", "3. Predict before you run code.\n", "4. Use failed checks as repair hints.\n", "5. Finish with the **Final Boss Evidence Memo**, where a score becomes an inspectable claim.\n", "\n", "Accessibility note: plot tasks ask for labels and legends because visual evidence should not rely on colour alone. The README includes conda setup, troubleshooting, and release notes.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "529e4d3b", "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", "import matplotlib.pyplot as plt\n", "\n", "SEED = 3151\n", "rng = np.random.default_rng(SEED)\n", "np.set_printoptions(precision=4, suppress=True)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "81316aad", "metadata": {}, "outputs": [], "source": [ "class Gradebook:\n", " def __init__(self):\n", " # Latest result for each named check. Rerunning a fixed check overwrites the old failure.\n", " self.results = {}\n", "\n", " def _truthy(self, condition):\n", " \"\"\"Safely convert scalars, NumPy scalars, and boolean arrays into one bool.\"\"\"\n", " try:\n", " arr = np.asarray(condition)\n", " if arr.shape == ():\n", " return bool(arr.item())\n", " return bool(np.all(arr))\n", " except Exception:\n", " return False\n", "\n", " def check(self, name, condition, hint=\"\"):\n", " ok = self._truthy(condition)\n", " self.results[name] = (ok, hint)\n", " print((\"✅\" if ok else \"❌\") + \" \" + name + (\"\" if ok or not hint else f\" — hint: {hint}\"))\n", " return ok\n", "\n", " def check_equal(self, name, got, expected, hint=\"\"):\n", " try:\n", " if isinstance(got, np.ndarray) or isinstance(expected, np.ndarray):\n", " ok = np.array_equal(got, expected)\n", " else:\n", " ok = got == expected\n", " except Exception:\n", " ok = False\n", " return self.check(name, ok, hint or f\"Expected {expected!r}, got {got!r}.\")\n", "\n", " def check_close(self, name, got, expected, tol=1e-8, hint=\"\"):\n", " try:\n", " ok = bool(np.all(np.isclose(got, expected, atol=tol, rtol=tol, equal_nan=True)))\n", " except Exception:\n", " ok = False\n", " return self.check(name, ok, hint or f\"Expected approximately {expected!r}, got {got!r}.\")\n", "\n", " def check_array_close(self, name, got, expected, tol=1e-8, hint=\"\"):\n", " try:\n", " got_arr = np.asarray(got)\n", " exp_arr = np.asarray(expected)\n", " ok = got_arr.shape == exp_arr.shape and np.allclose(got_arr, exp_arr, atol=tol, rtol=tol, equal_nan=True)\n", " except Exception:\n", " ok = False\n", " return self.check(name, ok, hint or f\"Expected array close to {expected!r}, got {got!r}.\")\n", "\n", " def check_shape(self, name, got, expected_shape, hint=\"\"):\n", " try:\n", " ok = np.asarray(got).shape == tuple(expected_shape)\n", " except Exception:\n", " ok = False\n", " return self.check(name, ok, hint or f\"Expected shape {expected_shape}, got {getattr(got, 'shape', None)}.\")\n", "\n", " def check_text(self, name, got, keywords=(), min_len=1, hint=\"\"):\n", " ok = isinstance(got, str) and len(got.strip()) >= min_len\n", " low = got.lower() if isinstance(got, str) else \"\"\n", " ok = ok and all(k.lower() in low for k in keywords)\n", " return self.check(name, ok, hint or f\"Write at least {min_len} characters and include: {keywords}.\")\n", "\n", " def summary(self):\n", " total = len(self.results)\n", " passed = sum(ok for ok, _ in self.results.values())\n", " print(f\"\\nPassed {passed}/{total} latest checks.\")\n", " failed = [name for name, (ok, _) in self.results.items() if not ok]\n", " if failed:\n", " print(\"Review these failed checks, fix your code, and rerun the relevant section:\")\n", " for name in failed:\n", " print(\" -\", name)\n", " else:\n", " print(\"Arcade cleared. You are ready to start the main 3151 arc.\")\n", "\n", "\n", "gb = Gradebook()\n", "\n", "\n", "def attempt(fn, *args, **kwargs):\n", " try:\n", " return fn(*args, **kwargs), None\n", " except Exception as e:\n", " return None, e\n", "\n", "\n", "def get(name):\n", " return globals().get(name, None)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "2966135c", "metadata": {}, "outputs": [], "source": [ "def plot_projection_frame(u, v, title=\"Projection picture\"):\n", " # Small visual helper used by the arcade. You do not need to edit this.\n", " u = np.asarray(u, dtype=float)\n", " v = np.asarray(v, dtype=float)\n", " proj = (u @ v) / (v @ v) * v\n", "\n", " fig, ax = plt.subplots(figsize=(5, 4))\n", " ax.axhline(0, linewidth=1)\n", " ax.axvline(0, linewidth=1)\n", " ax.arrow(0, 0, u[0], u[1], head_width=0.12, length_includes_head=True)\n", " ax.arrow(0, 0, v[0], v[1], head_width=0.12, length_includes_head=True)\n", " ax.arrow(0, 0, proj[0], proj[1], head_width=0.12, length_includes_head=True)\n", " ax.plot([u[0], proj[0]], [u[1], proj[1]], linestyle=\"--\", linewidth=1)\n", " ax.text(u[0] + 0.05, u[1] + 0.05, \"u\")\n", " ax.text(v[0] + 0.05, v[1] + 0.05, \"v\")\n", " ax.text(proj[0] + 0.05, proj[1] - 0.15, \"proj_v(u)\")\n", " lim = max(1.0, np.max(np.abs([u, v, proj])) + 0.6)\n", " ax.set_xlim(-lim, lim)\n", " ax.set_ylim(-lim, lim)\n", " ax.set_aspect(\"equal\", adjustable=\"box\")\n", " ax.set_title(title)\n", " return fig, ax\n", "\n", "\n", "def base_rate_tile_plot(N=1000, prevalence=0.01, sensitivity=0.90, specificity=0.91, seed=1109):\n", " # Natural-frequency tile plot inspired by the first-year data-science base-rate demo.\n", " rng_local = np.random.default_rng(seed)\n", " n_condition = int(round(N * prevalence))\n", " n_no = N - n_condition\n", " tp = int(round(n_condition * sensitivity))\n", " fn = n_condition - tp\n", " fp = int(round(n_no * (1 - specificity)))\n", " tn = n_no - fp\n", "\n", " cols = 40\n", " rows = int(np.ceil(N / cols))\n", " i = np.arange(N)\n", " x = i % cols\n", " y = i // cols\n", "\n", " condition_idx = rng_local.choice(i, size=n_condition, replace=False)\n", " tp_idx = rng_local.choice(condition_idx, size=tp, replace=False)\n", " fn_idx = np.setdiff1d(condition_idx, tp_idx)\n", " no_idx = np.setdiff1d(i, condition_idx)\n", " fp_idx = rng_local.choice(no_idx, size=fp, replace=False)\n", " positive_idx = np.sort(np.concatenate([tp_idx, fp_idx]))\n", "\n", " fig, ax = plt.subplots(figsize=(10, 5))\n", " ax.scatter(x, y, marker=\"s\", s=55, facecolors=\"none\", edgecolors=(0, 0, 0, 0.20), linewidths=0.5)\n", " ax.scatter(x[positive_idx], y[positive_idx], marker=\"s\", s=55, alpha=0.30, linewidths=0.5)\n", " ax.scatter(x[tp_idx], y[tp_idx], marker=\"s\", s=55, alpha=0.70, linewidths=0.5)\n", " ax.scatter(x[condition_idx], y[condition_idx], marker=\"x\", s=45, linewidths=1.1)\n", " ax.set_aspect(\"equal\")\n", " ax.set_xticks([])\n", " ax.set_yticks([])\n", " ax.set_xlim(-1, cols)\n", " ax.set_ylim(rows, -1)\n", " ax.set_title(f\"Positive tests: {tp + fp}; true positives: {tp}; posterior ≈ {tp/(tp+fp):.1%}\")\n", " return fig, ax\n" ] }, { "cell_type": "markdown", "id": "b54847f7", "metadata": {}, "source": [ "# Start Screen Diagnostic — low-stakes\n", "\n", "Complete this before Arcade 1. It is **not part of the Gradebook summary**. Use it to decide where to slow down.\n", "\n", "Fill in your best answers, then run the feedback cell. Wrong answers are useful: they tell you which arcade to treat as a repair zone.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "5ec3ce0d", "metadata": {}, "outputs": [], "source": [ "# TODO: fill these in before running the feedback cell.\n", "# Use tuples for shapes, booleans for true/false, and strings for split names.\n", "\n", "SD1_shape_Xw = None # If X.shape == (100, 3) and w.shape == (3,), what is (X @ w).shape?\n", "SD2_elementwise_same = None # True/False: X * w means the same thing as X @ w.\n", "SD3_fit_scaler_split = None # Which split should determine standardization statistics? \"train\", \"validation\", or \"test\"?\n", "SD4_positive_derivative = None # If f'(x) > 0 locally, is f usually \"increasing\" or \"decreasing\" near x?\n", "SD5_gradient_direction = None # The gradient points in the direction of steepest \"ascent\" or \"descent\"?\n", "SD6_conditional_denominator = None # In P(A | B), the denominator is P(A), P(B), or P(A and B)?\n", "SD7_high_accuracy_enough = None # True/False: 98% accuracy is automatically strong evidence.\n", "SD8_choose_k_split = None # Which split should choose k? \"train\", \"validation\", or \"test\"?\n", "SD9_final_once_split = None # Which split is saved for one final evaluation? \"train\", \"validation\", or \"test\"?\n", "SD10_score_without_baseline = None # True/False: a model score without a baseline is enough evidence.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "e325afde", "metadata": {}, "outputs": [], "source": [ "def _diag_text(x):\n", " return str(x).strip().lower().replace(\"_\", \" \")\n", "\n", "_diagnostic_checks = [\n", " (\"SD1 shape of X @ w\", SD1_shape_Xw == (100,)),\n", " (\"SD2 elementwise is not matrix-vector\", SD2_elementwise_same is False),\n", " (\"SD3 standardization stats come from train\", _diag_text(SD3_fit_scaler_split) in {\"train\", \"training\"}),\n", " (\"SD4 positive derivative means increasing\", _diag_text(SD4_positive_derivative) == \"increasing\"),\n", " (\"SD5 gradient points steepest ascent\", _diag_text(SD5_gradient_direction) in {\"ascent\", \"steepest ascent\", \"uphill\"}),\n", " (\"SD6 denominator of P(A | B)\", _diag_text(SD6_conditional_denominator) in {\"p(b)\", \"b\", \"p of b\", \"probability of b\"}),\n", " (\"SD7 high accuracy alone is not enough\", SD7_high_accuracy_enough is False),\n", " (\"SD8 validation chooses k\", _diag_text(SD8_choose_k_split) == \"validation\"),\n", " (\"SD9 test is final once\", _diag_text(SD9_final_once_split) == \"test\"),\n", " (\"SD10 baseline is needed\", SD10_score_without_baseline is False),\n", "]\n", "\n", "_diag_score = 0\n", "for name, ok in _diagnostic_checks:\n", " _diag_score += int(bool(ok))\n", " print((\"✅\" if ok else \"❌\"), name)\n", "\n", "print(f\"\\nStart Screen Diagnostic score: {_diag_score}/10\")\n", "if _diag_score <= 3:\n", " print(\"Suggested route: watch all videos carefully and complete every check.\")\n", "elif _diag_score <= 7:\n", " print(\"Suggested route: watch all videos, then slow down on failed notebook checks.\")\n", "else:\n", " print(\"Suggested route: move briskly, then use the Final Boss as your main readiness test.\")\n" ] }, { "cell_type": "markdown", "id": "4da74c29", "metadata": {}, "source": [ "# Arcade 1 — Vectors, Matrices, Dot Products, and Shape\n", "\n", "A vector is one measurement bundle. A matrix is many measurement bundles stacked together. A dot product turns two equally long vectors into one number.\n", "\n", "The main question in this arcade is: **what shape and meaning should the result have before you run the code?**\n" ] }, { "cell_type": "markdown", "id": "5caa6413", "metadata": {}, "source": [ "## A1. Dot products, norms, and cosine similarity\n", "\n", "Compute the dot product, the Euclidean norm of `u`, and the cosine similarity between `u` and `v`.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "7c22fe8c", "metadata": {}, "outputs": [], "source": [ "u = np.array([3.0, 4.0, 0.0])\n", "v = np.array([1.0, 0.0, 2.0])\n", "\n", "# TODO\n", "A1_dot = None\n", "A1_norm_u = None\n", "A1_cosine = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "4be0f7e4", "metadata": {}, "outputs": [], "source": [ "gb.check_close(\"A1.1 dot product\", get(\"A1_dot\"), 3.0, hint=\"Use u @ v or np.dot(u, v).\")\n", "gb.check_close(\"A1.2 norm\", get(\"A1_norm_u\"), 5.0, hint=\"Use sqrt(sum(u**2)) or np.linalg.norm(u).\")\n", "gb.check_close(\"A1.3 cosine\", get(\"A1_cosine\"), 3/(5*np.sqrt(5)), hint=\"Divide dot product by the product of the two norms.\")\n" ] }, { "cell_type": "markdown", "id": "c5a83811", "metadata": {}, "source": [ "## A2. Dot product as a transparent weighted score\n", "\n", "This is adapted from the 1109 “admissions as geometry” story, but here it is only a toy arithmetic exercise. We are **not** fitting a model and we are **not** recommending an admissions policy.\n", "\n", "One applicant is represented by `[gpa, test_score, activities]`. A hand-written scoring rule uses weights and a bias.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "96388778", "metadata": {}, "outputs": [], "source": [ "applicant_A2 = np.array([3.2, 80.0, 2.0])\n", "weights_A2 = np.array([18.0, 0.6, 2.5])\n", "bias_A2 = -20.0\n", "feature_names_A2 = np.array([\"gpa\", \"test_score\", \"activities\"])\n", "\n", "# TODO\n", "A2_contributions = None # elementwise feature contributions\n", "A2_score = None # weighted score plus bias\n", "A2_largest_contribution = None # feature name with largest positive contribution\n" ] }, { "cell_type": "code", "execution_count": null, "id": "4fc30132", "metadata": {}, "outputs": [], "source": [ "gb.check_array_close(\"A2.1 contributions\", get(\"A2_contributions\"), np.array([57.6, 48.0, 5.0]), hint=\"Multiply each feature by its matching weight.\")\n", "gb.check_close(\"A2.2 score\", get(\"A2_score\"), 90.6, hint=\"Compute applicant_A2 @ weights_A2 + bias_A2.\")\n", "gb.check_equal(\"A2.3 largest contributor\", get(\"A2_largest_contribution\"), \"gpa\", hint=\"Which feature contributes the largest number of points?\")\n" ] }, { "cell_type": "markdown", "id": "6308e3ce", "metadata": {}, "source": [ "## A3. Angle from a dot product\n", "\n", "For nonzero vectors, rearrange\n", "\n", "\\[\n", "\\cos(\\theta)=\\frac{u\\cdot v}{\\lVert u\\rVert\\lVert v\\rVert}.\n", "\\]\n" ] }, { "cell_type": "code", "execution_count": null, "id": "4620542f", "metadata": {}, "outputs": [], "source": [ "x_A3 = np.array([1.0, 0.0])\n", "y_A3 = np.array([0.5, np.sqrt(3)/2])\n", "\n", "# TODO\n", "A3_cos_theta = None\n", "A3_angle_degrees = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "3a390bce", "metadata": {}, "outputs": [], "source": [ "gb.check_close(\"A3.1 cosine\", get(\"A3_cos_theta\"), 0.5, tol=1e-10, hint=\"The vectors are unit length here.\")\n", "gb.check_close(\"A3.2 angle\", get(\"A3_angle_degrees\"), 60.0, tol=1e-8, hint=\"Use arccos, then convert radians to degrees.\")\n" ] }, { "cell_type": "markdown", "id": "01462c72", "metadata": {}, "source": [ "## A4. Projection\n", "\n", "Write a function for\n", "\n", "\\[\n", "\\operatorname{proj}_b(a)=\\frac{a\\cdot b}{b\\cdot b}b.\n", "\\]\n" ] }, { "cell_type": "code", "execution_count": null, "id": "65f2acf1", "metadata": {}, "outputs": [], "source": [ "def project_onto(a, b):\n", " # TODO\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "72f342a4", "metadata": {}, "outputs": [], "source": [ "fn = get(\"project_onto\")\n", "gb.check(\"A4.1 function callable\", callable(fn), hint=\"Define project_onto(a, b).\")\n", "if callable(fn):\n", " out1, _ = attempt(fn, np.array([3.0, 4.0]), np.array([2.0, 0.0]))\n", " out2, _ = attempt(fn, np.array([2.0, 1.0]), np.array([1.0, 1.0]))\n", "else:\n", " out1 = out2 = None\n", "gb.check_array_close(\"A4.2 projection onto x-axis\", out1, np.array([3.0, 0.0]), hint=\"The shadow of [3,4] on the x-axis is [3,0].\")\n", "gb.check_array_close(\"A4.3 projection onto diagonal\", out2, np.array([1.5, 1.5]), hint=\"Use the formula; do not assume b is unit length.\")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "e7e95971", "metadata": {}, "outputs": [], "source": [ "# Optional visual: run this after A4 works.\n", "# plot_projection_frame(np.array([3.0, 4.0]), np.array([2.0, 1.0]));\n" ] }, { "cell_type": "markdown", "id": "f569b0ee", "metadata": {}, "source": [ "## A5. Matrix-vector multiplication shapes\n", "\n", "If `X` has shape `(n, d)` and `w` has shape `(d,)`, then `X @ w` has shape `(n,)`: one score per row.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "120df06f", "metadata": {}, "outputs": [], "source": [ "X_A5 = np.array([\n", " [1.0, 0.0, 2.0],\n", " [0.0, 1.0, 1.0],\n", " [1.0, 1.0, 0.0],\n", " [2.0, -1.0, 1.0],\n", "])\n", "w_A5 = np.array([0.5, -2.0, 1.0])\n", "b_A5 = 1.5\n", "\n", "# TODO\n", "A5_scores = None\n", "A5_scores_shape = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "566eb689", "metadata": {}, "outputs": [], "source": [ "gb.check_array_close(\"A5.1 batch scores\", get(\"A5_scores\"), np.array([4.0, 0.5, 0.0, 5.5]), hint=\"Use X_A5 @ w_A5 + b_A5.\")\n", "gb.check_equal(\"A5.2 score shape\", get(\"A5_scores_shape\"), (4,), hint=\"One score per row.\")\n" ] }, { "cell_type": "markdown", "id": "5b8a1f9a", "metadata": {}, "source": [ "## A6. Shape brainteaser\n", "\n", "Answer these without trial-and-error. You may inspect `.shape`, but first say the shapes out loud.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "da6e750e", "metadata": {}, "outputs": [], "source": [ "X_A6 = np.ones((4, 3))\n", "w_A6 = np.array([2.0, 0.0, -1.0])\n", "mu_A6 = np.array([10.0, 20.0, 30.0])\n", "\n", "# TODO\n", "A6_inner_dimension = None\n", "A6_num_examples = None\n", "A6_shape_X_matmul_w = None\n", "A6_shape_X_minus_mu = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "1a45925a", "metadata": {}, "outputs": [], "source": [ "gb.check_equal(\"A6.1 inner dimension\", get(\"A6_inner_dimension\"), 3, hint=\"The 3 feature columns match the 3 weights.\")\n", "gb.check_equal(\"A6.2 number of examples\", get(\"A6_num_examples\"), 4, hint=\"Rows are examples.\")\n", "gb.check_equal(\"A6.3 X @ w shape\", get(\"A6_shape_X_matmul_w\"), (4,), hint=\"One scalar score per row.\")\n", "gb.check_equal(\"A6.4 X - mu shape\", get(\"A6_shape_X_minus_mu\"), (4, 3), hint=\"mu broadcasts across rows.\")\n" ] }, { "cell_type": "markdown", "id": "d23f32da", "metadata": {}, "source": [ "## A7. Add a bias column\n", "\n", "Some formulas absorb the bias/intercept into the feature matrix by adding a leading column of ones.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "a6a3bbc1", "metadata": {}, "outputs": [], "source": [ "def add_bias_column(X):\n", " # TODO\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "5b402372", "metadata": {}, "outputs": [], "source": [ "fn = get(\"add_bias_column\")\n", "gb.check(\"A7.1 function callable\", callable(fn), hint=\"Define add_bias_column(X).\")\n", "if callable(fn):\n", " out, _ = attempt(fn, np.array([[2.0, 3.0], [4.0, 5.0]]))\n", "else:\n", " out = None\n", "gb.check_array_close(\"A7.2 bias column\", out, np.array([[1.0, 2.0, 3.0], [1.0, 4.0, 5.0]]), hint=\"Put ones as the first column.\")\n" ] }, { "cell_type": "markdown", "id": "ac98b2a1", "metadata": {}, "source": [ "## A8. Standardize using training statistics\n", "\n", "The train set determines the mean and standard deviation. Validation/test data use those training values.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "783a88c8", "metadata": {}, "outputs": [], "source": [ "def fit_standardize(X_train, X_other):\n", " # TODO\n", " return None, None, None, None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "327bb83c", "metadata": {}, "outputs": [], "source": [ "Xtr_A8 = np.array([[1.0, 10.0], [3.0, 14.0], [5.0, 18.0]])\n", "Xte_A8 = np.array([[7.0, 22.0]])\n", "fn = get(\"fit_standardize\")\n", "gb.check(\"A8.1 function callable\", callable(fn), hint=\"Define fit_standardize(X_train, X_other).\")\n", "if callable(fn):\n", " out, err = attempt(fn, Xtr_A8, Xte_A8)\n", "else:\n", " out = None\n", "if isinstance(out, tuple) and len(out) == 4:\n", " Xtr_std_A8, Xte_std_A8, mu_A8, sigma_A8 = out\n", "else:\n", " Xtr_std_A8 = Xte_std_A8 = mu_A8 = sigma_A8 = None\n", "expected_sigma_A8 = np.array([np.std([1,3,5]), np.std([10,14,18])])\n", "gb.check_array_close(\"A8.2 training mean\", mu_A8, np.array([3.0, 14.0]), hint=\"Compute mean over axis=0 using training data only.\")\n", "gb.check_array_close(\"A8.3 training std\", sigma_A8, expected_sigma_A8, hint=\"Use np.std with ddof=0 for this exercise.\")\n", "gb.check_array_close(\"A8.4 standardized train mean near 0\", np.mean(Xtr_std_A8, axis=0) if Xtr_std_A8 is not None else None, np.array([0.0, 0.0]), tol=1e-10, hint=\"After standardizing train data, feature means should be 0.\")\n", "gb.check_array_close(\"A8.5 standardized test\", Xte_std_A8, (Xte_A8 - np.array([3.0,14.0]))/expected_sigma_A8, hint=\"Use training mean/std on the test data.\")\n" ] }, { "cell_type": "markdown", "id": "fdacb2b4", "metadata": {}, "source": [ "## A9. Broadcasting trap: column-centering versus row-centering\n", "\n", "Both of these can be legal NumPy, but they answer different questions.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "dcb00cc1", "metadata": {}, "outputs": [], "source": [ "X_A9 = np.array([[1.0, 10.0], [3.0, 14.0], [5.0, 18.0]])\n", "\n", "# TODO\n", "A9_column_centered_first_row = None\n", "A9_row_centered_first_row = None\n", "A9_explanation = \"\" # one sentence: what changed?\n" ] }, { "cell_type": "code", "execution_count": null, "id": "0b86b7d0", "metadata": {}, "outputs": [], "source": [ "gb.check_array_close(\"A9.1 column-centered first row\", get(\"A9_column_centered_first_row\"), np.array([-2.0, -4.0]), hint=\"Subtract the mean of each feature column.\")\n", "gb.check_array_close(\"A9.2 row-centered first row\", get(\"A9_row_centered_first_row\"), np.array([-4.5, 4.5]), hint=\"Subtract the first row's own mean from both entries.\")\n", "gb.check_text(\"A9.3 explanation\", get(\"A9_explanation\"), keywords=(\"column\", \"row\"), min_len=50, hint=\"Mention column-centering and row-centering.\")\n" ] }, { "cell_type": "markdown", "id": "833d4246", "metadata": {}, "source": [ "## A10. Pairwise dot products as a Gram matrix\n", "\n", "For a matrix `Z`, `Z @ Z.T` contains every row-vector dot product against every other row-vector. Here it is just shape and dot-product practice.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "0147755d", "metadata": {}, "outputs": [], "source": [ "Z_A10 = np.array([[1.0, 0.0], [0.0, 1.0], [1.0, 1.0]])\n", "\n", "# TODO\n", "A10_gram = None\n", "A10_diagonal = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "f6e2f0c6", "metadata": {}, "outputs": [], "source": [ "gb.check_array_close(\"A10.1 Gram matrix\", get(\"A10_gram\"), np.array([[1.0, 0.0, 1.0], [0.0, 1.0, 1.0], [1.0, 1.0, 2.0]]), hint=\"Use Z_A10 @ Z_A10.T.\")\n", "gb.check_array_close(\"A10.2 diagonal contains squared norms\", get(\"A10_diagonal\"), np.array([1.0, 1.0, 2.0]), hint=\"A row dotted with itself is its squared length.\")\n" ] }, { "cell_type": "markdown", "id": "89202dd3", "metadata": {}, "source": [ "## A11. Operator quiz\n", "\n", "Use a string answer.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "a32649db", "metadata": {}, "outputs": [], "source": [ "# TODO: choose one of \"*\", \"@\", \"+\", or \"mean\".\n", "A11_operator_for_one_score_per_row = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "5116cace", "metadata": {}, "outputs": [], "source": [ "gb.check_equal(\"A11.1 right operator\", get(\"A11_operator_for_one_score_per_row\"), \"@\", hint=\"Matrix multiplication gives one dot product per row.\")\n" ] }, { "cell_type": "markdown", "id": "c45dc088", "metadata": {}, "source": [ "# Arcade 2 — Calculus and Local Change\n", "\n", "The goal is not to optimize anything yet. The goal is to read derivatives as **local change information** so later optimization material has somewhere to land.\n" ] }, { "cell_type": "markdown", "id": "8966e103", "metadata": {}, "source": [ "## B1. Scalar derivative\n" ] }, { "cell_type": "code", "execution_count": null, "id": "b5bab0d0", "metadata": {}, "outputs": [], "source": [ "def f_B1(x):\n", " return 3*x**2 - 2*x + 5\n", "\n", "# TODO: derivative of f_B1\n", "\n", "def fprime_B1(x):\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "04128082", "metadata": {}, "outputs": [], "source": [ "fn = get(\"fprime_B1\")\n", "gb.check(\"B1.1 function callable\", callable(fn), hint=\"Define fprime_B1(x).\")\n", "if callable(fn):\n", " gb.check_close(\"B1.2 derivative at 0\", fn(0.0), -2.0)\n", " gb.check_close(\"B1.3 derivative at 3\", fn(3.0), 16.0)\n" ] }, { "cell_type": "markdown", "id": "6351d533", "metadata": {}, "source": [ "## B2. Polynomial derivative at a point\n" ] }, { "cell_type": "code", "execution_count": null, "id": "0e566a9c", "metadata": {}, "outputs": [], "source": [ "def h_B2(x):\n", " return x**3 - 4*x\n", "\n", "# TODO\n", "B2_hprime_at_2 = None\n", "B2_h_increasing_at_2 = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "d78f6360", "metadata": {}, "outputs": [], "source": [ "gb.check_close(\"B2.1 derivative at 2\", get(\"B2_hprime_at_2\"), 8.0, hint=\"h'(x)=3x^2-4.\")\n", "gb.check_equal(\"B2.2 increasing at 2\", get(\"B2_h_increasing_at_2\"), True, hint=\"Positive derivative means locally increasing.\")\n" ] }, { "cell_type": "markdown", "id": "cb872d53", "metadata": {}, "source": [ "## B3. Partial derivatives and gradients\n" ] }, { "cell_type": "code", "execution_count": null, "id": "a49b3b1e", "metadata": {}, "outputs": [], "source": [ "def g_B3(a, b):\n", " return a**2 * b + np.sin(b)\n", "\n", "# TODO: return np.array([partial g / partial a, partial g / partial b])\n", "def grad_g_B3(a, b):\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "28642f21", "metadata": {}, "outputs": [], "source": [ "fn = get(\"grad_g_B3\")\n", "gb.check(\"B3.1 function callable\", callable(fn), hint=\"Define grad_g_B3(a, b).\")\n", "if callable(fn):\n", " gb.check_array_close(\"B3.2 gradient at (2,0)\", fn(2.0, 0.0), np.array([0.0, 5.0]), hint=\"Differentiate with respect to a and b separately.\")\n", " gb.check_array_close(\"B3.3 gradient at (1,pi)\", fn(1.0, np.pi), np.array([2*np.pi, 0.0]), tol=1e-8)\n" ] }, { "cell_type": "markdown", "id": "341a7bbf", "metadata": {}, "source": [ "## B4. Directional derivative\n" ] }, { "cell_type": "code", "execution_count": null, "id": "483708a4", "metadata": {}, "outputs": [], "source": [ "def directional_derivative_B4(a, b, direction):\n", " # TODO: use grad_g_B3 and the unit direction\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "3bec5099", "metadata": {}, "outputs": [], "source": [ "fn = get(\"directional_derivative_B4\")\n", "gb.check(\"B4.1 function callable\", callable(fn), hint=\"Define directional_derivative_B4(a, b, direction).\")\n", "if callable(fn):\n", " expected = np.array([2*1.0*2.0, 1.0**2 + np.cos(2.0)]) @ (np.array([3.0, 4.0]) / 5.0)\n", " got, _ = attempt(fn, 1.0, 2.0, np.array([3.0, 4.0]))\n", " gb.check_close(\"B4.2 directional derivative\", got, expected, hint=\"Normalize the direction before taking a dot product.\")\n" ] }, { "cell_type": "markdown", "id": "7fa12321", "metadata": {}, "source": [ "## B5. Centered finite difference\n" ] }, { "cell_type": "code", "execution_count": null, "id": "ed846481", "metadata": {}, "outputs": [], "source": [ "def finite_difference_1d(f, x, h=1e-5):\n", " # TODO\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "aee9095d", "metadata": {}, "outputs": [], "source": [ "fn = get(\"finite_difference_1d\")\n", "gb.check(\"B5.1 function callable\", callable(fn), hint=\"Define finite_difference_1d(f, x, h).\")\n", "if callable(fn):\n", " gb.check_close(\"B5.2 x^2 at 3\", fn(lambda x: x**2, 3.0), 6.0, tol=1e-4)\n", " gb.check_close(\"B5.3 sin at 0\", fn(np.sin, 0.0), 1.0, tol=1e-5)\n" ] }, { "cell_type": "markdown", "id": "df95f0ab", "metadata": {}, "source": [ "## B6. Sigmoid and its derivative\n" ] }, { "cell_type": "code", "execution_count": null, "id": "0b526b94", "metadata": {}, "outputs": [], "source": [ "def sigmoid(z):\n", " # TODO\n", " return None\n", "\n", "def sigmoid_derivative(z):\n", " # TODO\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "01d48287", "metadata": {}, "outputs": [], "source": [ "fn_sig = get(\"sigmoid\")\n", "fn_der = get(\"sigmoid_derivative\")\n", "gb.check(\"B6.1 sigmoid callable\", callable(fn_sig), hint=\"Define sigmoid(z).\")\n", "gb.check(\"B6.2 derivative callable\", callable(fn_der), hint=\"Define sigmoid_derivative(z).\")\n", "if callable(fn_sig):\n", " gb.check_close(\"B6.3 sigmoid(0)\", fn_sig(0.0), 0.5)\n", " gb.check_array_close(\"B6.4 sigmoid vector\", fn_sig(np.array([-1.0, 0.0, 1.0])), 1/(1+np.exp(-np.array([-1.0,0.0,1.0]))))\n", "if callable(fn_der):\n", " gb.check_close(\"B6.5 derivative at 0\", fn_der(0.0), 0.25)\n" ] }, { "cell_type": "markdown", "id": "dd86edb2", "metadata": {}, "source": [ "## B7. Tangent-line calculator\n" ] }, { "cell_type": "code", "execution_count": null, "id": "42fd6ba7", "metadata": {}, "outputs": [], "source": [ "def tangent_line_values(f, fprime, x0, xs):\n", " # TODO: f(x0) + fprime(x0) * (xs - x0)\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "acfbed1f", "metadata": {}, "outputs": [], "source": [ "fn = get(\"tangent_line_values\")\n", "gb.check(\"B7.1 function callable\", callable(fn), hint=\"Define tangent_line_values(f, fprime, x0, xs).\")\n", "if callable(fn):\n", " xs = np.array([1.0, 2.0, 3.0])\n", " gb.check_array_close(\"B7.2 tangent to x^2 at 2\", fn(lambda x: x**2, lambda x: 2*x, 2.0, xs), np.array([0.0, 4.0, 8.0]), hint=\"The tangent line is 4 + 4(x-2).\")\n" ] }, { "cell_type": "markdown", "id": "c0c18784", "metadata": {}, "source": [ "## B8. Local-change explanation\n" ] }, { "cell_type": "code", "execution_count": null, "id": "f5d52af6", "metadata": {}, "outputs": [], "source": [ "# TODO: 2-4 sentences. Explain derivative vs optimization algorithm.\n", "B8_explanation = \"\"\n" ] }, { "cell_type": "code", "execution_count": null, "id": "431e1466", "metadata": {}, "outputs": [], "source": [ "gb.check_text(\"B8.1 explanation\", get(\"B8_explanation\"), keywords=(\"derivative\", \"algorithm\"), min_len=120, hint=\"Contrast local change information with a procedure that repeatedly updates something.\")\n" ] }, { "cell_type": "markdown", "id": "569f2f08", "metadata": {}, "source": [ "# Arcade 3 — Probability, Base Rates, and Distributions\n", "\n", "The point is to reason with counts, conditions, and uncertainty before the course asks you to evaluate models under uncertainty.\n" ] }, { "cell_type": "markdown", "id": "3490016c", "metadata": {}, "source": [ "## C1. Probability arithmetic\n" ] }, { "cell_type": "code", "execution_count": null, "id": "9b1b1978", "metadata": {}, "outputs": [], "source": [ "P_A = 0.30\n", "P_B = 0.40\n", "P_A_and_B = 0.18\n", "\n", "# TODO\n", "C1_not_A = None\n", "C1_A_or_B = None\n", "C1_are_independent = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "00cb899d", "metadata": {}, "outputs": [], "source": [ "gb.check_close(\"C1.1 complement\", get(\"C1_not_A\"), 0.70)\n", "gb.check_close(\"C1.2 union\", get(\"C1_A_or_B\"), 0.52, hint=\"P(A or B)=P(A)+P(B)-P(A and B).\")\n", "gb.check_equal(\"C1.3 independence\", get(\"C1_are_independent\"), False, hint=\"For independence, P(A and B) would equal P(A)P(B).\")\n" ] }, { "cell_type": "markdown", "id": "531fd7f5", "metadata": {}, "source": [ "## C2. Conditional probability from a table\n" ] }, { "cell_type": "code", "execution_count": null, "id": "b062307d", "metadata": {}, "outputs": [], "source": [ "# Rows are true labels [0, 1]; columns are predicted labels [0, 1].\n", "conf_C2 = np.array([[50, 10],\n", " [ 5, 35]])\n", "\n", "# TODO\n", "C2_P_true1_given_pred1 = None\n", "C2_P_pred1_given_true1 = None\n", "C2_total_errors = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "f9622824", "metadata": {}, "outputs": [], "source": [ "gb.check_close(\"C2.1 P(true=1 | pred=1)\", get(\"C2_P_true1_given_pred1\"), 35/45, hint=\"Condition on the predicted-positive column.\")\n", "gb.check_close(\"C2.2 P(pred=1 | true=1)\", get(\"C2_P_pred1_given_true1\"), 35/40, hint=\"Condition on the true-positive row.\")\n", "gb.check_equal(\"C2.3 total errors\", get(\"C2_total_errors\"), 15, hint=\"Off-diagonal entries are errors.\")\n" ] }, { "cell_type": "markdown", "id": "979fd2e2", "metadata": {}, "source": [ "## C3. Bayes' rule: base rates matter\n", "\n", "A screening test has prevalence 1%, sensitivity 90%, and specificity 91%. What is the probability of the condition after a positive test?\n" ] }, { "cell_type": "code", "execution_count": null, "id": "2ffb981f", "metadata": {}, "outputs": [], "source": [ "prevalence_C3 = 0.01\n", "sensitivity_C3 = 0.90\n", "specificity_C3 = 0.91\n", "\n", "# TODO\n", "C3_p_positive = None\n", "C3_p_condition_given_positive = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "29b95fcf", "metadata": {}, "outputs": [], "source": [ "expected_C3 = 0.009 / (0.009 + 0.0891)\n", "gb.check_close(\"C3.1 P(positive)\", get(\"C3_p_positive\"), 0.0981, hint=\"Positive can mean true positive or false positive.\")\n", "gb.check_close(\"C3.2 posterior\", get(\"C3_p_condition_given_positive\"), expected_C3, hint=\"Posterior = true-positive probability / all-positive probability.\")\n" ] }, { "cell_type": "markdown", "id": "f6d0a8aa", "metadata": {}, "source": [ "## C4. Natural-frequency counts\n" ] }, { "cell_type": "code", "execution_count": null, "id": "a0766f50", "metadata": {}, "outputs": [], "source": [ "# Use N=1000 with the same screening assumptions as C3.\n", "# TODO\n", "C4_true_positives = None\n", "C4_false_negatives = None\n", "C4_false_positives = None\n", "C4_true_negatives = None\n", "C4_positive_predictive_value = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "77775833", "metadata": {}, "outputs": [], "source": [ "gb.check_equal(\"C4.1 true positives\", get(\"C4_true_positives\"), 9)\n", "gb.check_equal(\"C4.2 false negatives\", get(\"C4_false_negatives\"), 1)\n", "gb.check_equal(\"C4.3 false positives\", get(\"C4_false_positives\"), 89)\n", "gb.check_equal(\"C4.4 true negatives\", get(\"C4_true_negatives\"), 901)\n", "gb.check_close(\"C4.5 positive predictive value\", get(\"C4_positive_predictive_value\"), 9/98)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "f4beea0c", "metadata": {}, "outputs": [], "source": [ "# Optional visual: a 1000-person tile grid. Run this cell to see why base rates matter.\n", "# base_rate_tile_plot();\n" ] }, { "cell_type": "markdown", "id": "8546573e", "metadata": {}, "source": [ "## C5. Expected value and variance\n" ] }, { "cell_type": "code", "execution_count": null, "id": "6f2f8f5d", "metadata": {}, "outputs": [], "source": [ "values_C5 = np.array([0.0, 1.0, 2.0, 5.0])\n", "probs_C5 = np.array([0.2, 0.3, 0.4, 0.1])\n", "\n", "# TODO\n", "C5_expected_value = None\n", "C5_variance = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "61ddccda", "metadata": {}, "outputs": [], "source": [ "gb.check_close(\"C5.1 expected value\", get(\"C5_expected_value\"), 1.6, hint=\"Sum value * probability.\")\n", "gb.check_close(\"C5.2 variance\", get(\"C5_variance\"), 1.84, hint=\"Sum squared deviations times probabilities.\")\n" ] }, { "cell_type": "markdown", "id": "1a67e9a9", "metadata": {}, "source": [ "## C6. Sample statistics\n" ] }, { "cell_type": "code", "execution_count": null, "id": "ede92954", "metadata": {}, "outputs": [], "source": [ "def sample_summary(x):\n", " # TODO: return dict with mean, median, std_pop, q1, q3, iqr\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "a580b210", "metadata": {}, "outputs": [], "source": [ "fn = get(\"sample_summary\")\n", "gb.check(\"C6.1 function callable\", callable(fn), hint=\"Define sample_summary(x).\")\n", "summary_C6 = fn(np.array([2.0, 4.0, 4.0, 10.0])) if callable(fn) else None\n", "gb.check(\"C6.2 returns dict\", isinstance(summary_C6, dict), hint=\"Return a dictionary.\")\n", "if isinstance(summary_C6, dict):\n", " gb.check_close(\"C6.3 mean\", summary_C6.get(\"mean\"), 5.0)\n", " gb.check_close(\"C6.4 median\", summary_C6.get(\"median\"), 4.0)\n", " gb.check_close(\"C6.5 std_pop\", summary_C6.get(\"std_pop\"), np.std([2,4,4,10]))\n", " gb.check_close(\"C6.6 iqr\", summary_C6.get(\"iqr\"), np.percentile([2,4,4,10],75)-np.percentile([2,4,4,10],25))\n" ] }, { "cell_type": "markdown", "id": "555a4940", "metadata": {}, "source": [ "## C7. Same mean, different distribution shape\n" ] }, { "cell_type": "code", "execution_count": null, "id": "259b19ed", "metadata": {}, "outputs": [], "source": [ "class_A_C7 = np.array([69, 70, 71, 68, 72, 70, 69, 71])\n", "class_B_C7 = np.array([20, 30, 40, 70, 100, 100, 100, 100])\n", "\n", "# TODO\n", "C7_mean_A = None\n", "C7_mean_B = None\n", "C7_range_A = None\n", "C7_range_B = None\n", "C7_same_story = None # True/False: does same mean imply same story?\n" ] }, { "cell_type": "code", "execution_count": null, "id": "751576e4", "metadata": {}, "outputs": [], "source": [ "gb.check_close(\"C7.1 mean A\", get(\"C7_mean_A\"), 70.0)\n", "gb.check_close(\"C7.2 mean B\", get(\"C7_mean_B\"), 70.0)\n", "gb.check_equal(\"C7.3 range A\", get(\"C7_range_A\"), 4)\n", "gb.check_equal(\"C7.4 range B\", get(\"C7_range_B\"), 80)\n", "gb.check_equal(\"C7.5 same story?\", get(\"C7_same_story\"), False, hint=\"A single summary can hide shape and spread.\")\n" ] }, { "cell_type": "markdown", "id": "4b267e89", "metadata": {}, "source": [ "## C8. IQR outlier rule\n" ] }, { "cell_type": "code", "execution_count": null, "id": "ace17bc1", "metadata": {}, "outputs": [], "source": [ "def iqr_outlier_mask(x):\n", " # TODO: True for values below Q1-1.5*IQR or above Q3+1.5*IQR\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "1ff3b54f", "metadata": {}, "outputs": [], "source": [ "fn = get(\"iqr_outlier_mask\")\n", "gb.check(\"C8.1 function callable\", callable(fn), hint=\"Define iqr_outlier_mask(x).\")\n", "if callable(fn):\n", " gb.check_array_close(\"C8.2 one high outlier\", fn(np.array([10, 11, 12, 13, 14, 100])), np.array([False, False, False, False, False, True]))\n", " gb.check_array_close(\"C8.3 no outliers\", fn(np.array([1, 2, 3, 4, 5])), np.array([False, False, False, False, False]))\n" ] }, { "cell_type": "markdown", "id": "a0d65704", "metadata": {}, "source": [ "## C9. Rule-based outlier flags\n" ] }, { "cell_type": "code", "execution_count": null, "id": "e60079f2", "metadata": {}, "outputs": [], "source": [ "hr_C9 = np.array([68, 72, 0, 240, 130, 88, 31, 140])\n", "\n", "# TODO: flag values below 35 or above 120\n", "C9_flagged_values = None\n", "C9_num_flagged = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "1068d85c", "metadata": {}, "outputs": [], "source": [ "gb.check_array_close(\"C9.1 flagged values\", get(\"C9_flagged_values\"), np.array([0, 240, 130, 31, 140]), hint=\"Use a boolean mask with | for OR.\")\n", "gb.check_equal(\"C9.2 number flagged\", get(\"C9_num_flagged\"), 5)\n" ] }, { "cell_type": "markdown", "id": "a5f1c6eb", "metadata": {}, "source": [ "## C10. Split discipline\n" ] }, { "cell_type": "code", "execution_count": null, "id": "df32ebf1", "metadata": {}, "outputs": [], "source": [ "n_total_C10 = 1200\n", "\n", "# TODO: 60% train, 20% validation, 20% test\n", "C10_train_n = None\n", "C10_val_n = None\n", "C10_test_n = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "7bf4db85", "metadata": {}, "outputs": [], "source": [ "gb.check_equal(\"C10.1 train count\", get(\"C10_train_n\"), 720)\n", "gb.check_equal(\"C10.2 validation count\", get(\"C10_val_n\"), 240)\n", "gb.check_equal(\"C10.3 test count\", get(\"C10_test_n\"), 240)\n" ] }, { "cell_type": "markdown", "id": "5bc57293", "metadata": {}, "source": [ "## C11. Leakage explanation\n" ] }, { "cell_type": "code", "execution_count": null, "id": "3a807c49", "metadata": {}, "outputs": [], "source": [ "# TODO: 2-3 sentences. Why is it suspicious if test performance improves after many test-set peeks?\n", "C11_leakage_explanation = \"\"\n" ] }, { "cell_type": "code", "execution_count": null, "id": "df4ada6e", "metadata": {}, "outputs": [], "source": [ "gb.check_text(\"C11.1 leakage explanation\", get(\"C11_leakage_explanation\"), keywords=(\"test\", \"leak\"), min_len=120)\n" ] }, { "cell_type": "markdown", "id": "77f5afd0", "metadata": {}, "source": [ "## C12. Events as boolean masks\n", "\n", "A row-level dataset is often a sample space where events are boolean masks. Use the same 20 claw-machine plays from the slides.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "2ab9a912", "metadata": {}, "outputs": [], "source": [ "closes_C12 = np.array([True] * 12 + [False] * 8)\n", "wins_C12 = np.array([True] * 8 + [False] * 4 + [True] * 2 + [False] * 6)\n", "\n", "# TODO\n", "C12_P_wins = None\n", "C12_P_closes = None\n", "C12_P_wins_and_closes = None\n", "C12_P_wins_or_closes = None\n", "C12_P_wins_given_closes = None\n", "C12_P_closes_given_wins = None\n", "C12_neither_count = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "f6026cb5", "metadata": {}, "outputs": [], "source": [ "gb.check_close(\"C12.1 P(wins)\", get(\"C12_P_wins\"), 10/20)\n", "gb.check_close(\"C12.2 P(closes)\", get(\"C12_P_closes\"), 12/20)\n", "gb.check_close(\"C12.3 P(wins and closes)\", get(\"C12_P_wins_and_closes\"), 8/20)\n", "gb.check_close(\"C12.4 P(wins or closes)\", get(\"C12_P_wins_or_closes\"), 14/20, hint=\"Use OR, then average the boolean mask.\")\n", "gb.check_close(\"C12.5 P(wins | closes)\", get(\"C12_P_wins_given_closes\"), 8/12, hint=\"Condition by selecting the rows where closes_C12 is True.\")\n", "gb.check_close(\"C12.6 P(closes | wins)\", get(\"C12_P_closes_given_wins\"), 8/10)\n", "gb.check_equal(\"C12.7 neither count\", get(\"C12_neither_count\"), 6)\n" ] }, { "cell_type": "markdown", "id": "4643a66d", "metadata": {}, "source": [ "## C13. Bayes' rule as a reusable function\n", "\n", "Write a function for the posterior probability after a positive test.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "c740d8c9", "metadata": {}, "outputs": [], "source": [ "def posterior_after_positive(prevalence, sensitivity, specificity):\n", " # TODO\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "5fd6c269", "metadata": {}, "outputs": [], "source": [ "fn = get(\"posterior_after_positive\")\n", "gb.check(\"C13.1 function callable\", callable(fn), hint=\"Define posterior_after_positive(prevalence, sensitivity, specificity).\")\n", "if callable(fn):\n", " out1, _ = attempt(fn, 0.01, 0.90, 0.91)\n", " out2, _ = attempt(fn, 0.10, 0.90, 0.91)\n", " out3, _ = attempt(fn, 0.01, 0.99, 0.99)\n", "else:\n", " out1 = out2 = out3 = None\n", "gb.check_close(\"C13.2 rare condition\", out1, 0.009 / (0.009 + 0.99 * 0.09))\n", "gb.check_close(\"C13.3 less rare condition\", out2, 0.09 / (0.09 + 0.90 * 0.09))\n", "gb.check_close(\"C13.4 stronger test\", out3, 0.0099 / (0.0099 + 0.99 * 0.01))\n" ] }, { "cell_type": "markdown", "id": "50ace66d", "metadata": {}, "source": [ "## C14. Thresholds, confusion counts, precision, and recall\n", "\n", "A model score becomes a positive/negative decision only after you choose a threshold.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "a576217f", "metadata": {}, "outputs": [], "source": [ "y_true_C14 = np.array([1, 1, 1, 1, 0, 0, 0, 0, 1, 0])\n", "scores_C14 = np.array([0.95, 0.85, 0.70, 0.40, 0.65, 0.55, 0.30, 0.20, 0.45, 0.10])\n", "threshold_C14 = 0.60\n", "\n", "# TODO\n", "C14_y_pred = None\n", "C14_TP = None\n", "C14_FP = None\n", "C14_FN = None\n", "C14_TN = None\n", "C14_precision = None\n", "C14_recall = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "cd0c6ccc", "metadata": {}, "outputs": [], "source": [ "gb.check_array_close(\"C14.1 predictions\", get(\"C14_y_pred\"), np.array([1, 1, 1, 0, 1, 0, 0, 0, 0, 0]))\n", "gb.check_equal(\"C14.2 TP\", get(\"C14_TP\"), 3)\n", "gb.check_equal(\"C14.3 FP\", get(\"C14_FP\"), 1)\n", "gb.check_equal(\"C14.4 FN\", get(\"C14_FN\"), 2)\n", "gb.check_equal(\"C14.5 TN\", get(\"C14_TN\"), 4)\n", "gb.check_close(\"C14.6 precision\", get(\"C14_precision\"), 3/4, hint=\"TP / predicted positives.\")\n", "gb.check_close(\"C14.7 recall\", get(\"C14_recall\"), 3/5, hint=\"TP / actual positives.\")\n" ] }, { "cell_type": "markdown", "id": "fe8e9d25", "metadata": {}, "source": [ "## C15. A reported test score has sampling uncertainty\n", "\n", "Even if a model has a fixed true accuracy, the measured accuracy depends on how many test cases you happened to sample.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "f6a966f5", "metadata": {}, "outputs": [], "source": [ "true_accuracy_C15 = 0.80\n", "test_sizes_C15 = np.array([50, 200, 1000])\n", "\n", "# TODO\n", "C15_accuracy_sd = None # array: sqrt(p(1-p)/n) for each test size\n", "C15_approx_95_half_width = None # array: 1.96 * C15_accuracy_sd\n", "C15_most_stable_test_size = None # one of the values in test_sizes_C15\n" ] }, { "cell_type": "code", "execution_count": null, "id": "170e992f", "metadata": {}, "outputs": [], "source": [ "expected_sd_C15 = np.sqrt(true_accuracy_C15 * (1 - true_accuracy_C15) / test_sizes_C15)\n", "gb.check_array_close(\"C15.1 accuracy standard deviations\", get(\"C15_accuracy_sd\"), expected_sd_C15)\n", "gb.check_array_close(\"C15.2 approximate 95% half-widths\", get(\"C15_approx_95_half_width\"), 1.96 * expected_sd_C15)\n", "gb.check_equal(\"C15.3 most stable test size\", get(\"C15_most_stable_test_size\"), 1000, hint=\"Larger test sets usually make the estimate less wobbly.\")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "b5c9da4c", "metadata": {}, "outputs": [], "source": [ "# Optional visual after C15 works: simulated measured accuracies from the same true accuracy.\n", "rng_C15 = np.random.default_rng(3151)\n", "for n in test_sizes_C15:\n", " observed = rng_C15.binomial(n, true_accuracy_C15, size=5000) / n\n", " lo, hi = np.quantile(observed, [0.05, 0.95])\n", " print(f\"n={n:4d}: middle 90% of observed accuracies is about {lo:.2f} to {hi:.2f}\")\n" ] }, { "cell_type": "markdown", "id": "260ce6e5", "metadata": {}, "source": [ "## C16. Split base rates are probabilities too\n", "\n", "For rare labels, a split can accidentally change the base rate across train, validation, and test. This exercise uses a deliberately sorted label vector to make the danger visible.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "7daef9dc", "metadata": {}, "outputs": [], "source": [ "n_total_C16 = 1200\n", "n_positive_C16 = 60\n", "train_n_C16 = 720\n", "val_n_C16 = 240\n", "test_n_C16 = 240\n", "\n", "# Imagine the 60 positives come first because the rows are sorted by outcome.\n", "# TODO\n", "C16_overall_positive_rate = None\n", "C16_bad_train_positive_rate = None\n", "C16_bad_val_positive_rate = None\n", "C16_bad_test_positive_rate = None\n", "\n", "# For a stratified 60/20/20 split, preserve the 5% positive rate in each split.\n", "C16_strat_train_pos = None\n", "C16_strat_val_pos = None\n", "C16_strat_test_pos = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "5584c80d", "metadata": {}, "outputs": [], "source": [ "gb.check_close(\"C16.1 overall positive rate\", get(\"C16_overall_positive_rate\"), 60/1200)\n", "gb.check_close(\"C16.2 bad train positive rate\", get(\"C16_bad_train_positive_rate\"), 60/720)\n", "gb.check_close(\"C16.3 bad validation positive rate\", get(\"C16_bad_val_positive_rate\"), 0.0)\n", "gb.check_close(\"C16.4 bad test positive rate\", get(\"C16_bad_test_positive_rate\"), 0.0)\n", "gb.check_equal(\"C16.5 stratified train positives\", get(\"C16_strat_train_pos\"), 36)\n", "gb.check_equal(\"C16.6 stratified validation positives\", get(\"C16_strat_val_pos\"), 12)\n", "gb.check_equal(\"C16.7 stratified test positives\", get(\"C16_strat_test_pos\"), 12)\n" ] }, { "cell_type": "markdown", "id": "7f5f11ba", "metadata": {}, "source": [ "# Arcade 4 — Python, NumPy, Vectorization, and Small Algorithms\n", "\n", "Write code that matches the math. Prefer vectorized array operations when the meaning is naturally an array operation, but do not be afraid of a short clear loop when it is the simplest correct expression.\n" ] }, { "cell_type": "markdown", "id": "b722883a", "metadata": {}, "source": [ "## D1. Vectorized sum of squares\n" ] }, { "cell_type": "code", "execution_count": null, "id": "153b9c3e", "metadata": {}, "outputs": [], "source": [ "def sum_of_squares(xs):\n", " # TODO\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "55f27126", "metadata": {}, "outputs": [], "source": [ "fn = get(\"sum_of_squares\")\n", "gb.check(\"D1.1 function callable\", callable(fn), hint=\"Define sum_of_squares(xs).\")\n", "if callable(fn):\n", " gb.check_close(\"D1.2 sum\", fn(np.array([1.0, -2.0, 3.0])), 14.0)\n" ] }, { "cell_type": "markdown", "id": "7fb65523", "metadata": {}, "source": [ "## D2. Column means and centered data\n" ] }, { "cell_type": "code", "execution_count": null, "id": "7751d924", "metadata": {}, "outputs": [], "source": [ "def center_columns(X):\n", " # TODO: return centered_X, column_means\n", " return None, None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "cb898750", "metadata": {}, "outputs": [], "source": [ "fn = get(\"center_columns\")\n", "gb.check(\"D2.1 function callable\", callable(fn), hint=\"Define center_columns(X).\")\n", "if callable(fn):\n", " out, err = attempt(fn, np.array([[1.0, 10.0], [3.0, 14.0], [5.0, 18.0]]))\n", "else:\n", " out = None\n", "if isinstance(out, tuple) and len(out) == 2:\n", " centered_D2, mu_D2 = out\n", "else:\n", " centered_D2 = mu_D2 = None\n", "gb.check_array_close(\"D2.2 means\", mu_D2, np.array([3.0, 14.0]))\n", "gb.check_array_close(\"D2.3 centered\", centered_D2, np.array([[-2.0, -4.0], [0.0, 0.0], [2.0, 4.0]]))\n" ] }, { "cell_type": "markdown", "id": "a4452225", "metadata": {}, "source": [ "## D3. Boolean masks\n" ] }, { "cell_type": "code", "execution_count": null, "id": "44b6e527", "metadata": {}, "outputs": [], "source": [ "def rows_with_label_one(X, y):\n", " # TODO\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "4cc7db5c", "metadata": {}, "outputs": [], "source": [ "X_D3 = np.array([[0, 0], [1, 0], [0, 1], [1, 1]])\n", "y_D3 = np.array([0, 1, 0, 1])\n", "fn = get(\"rows_with_label_one\")\n", "gb.check(\"D3.1 function callable\", callable(fn), hint=\"Define rows_with_label_one(X, y).\")\n", "if callable(fn):\n", " gb.check_array_close(\"D3.2 selected rows\", fn(X_D3, y_D3), np.array([[1,0], [1,1]]))\n" ] }, { "cell_type": "markdown", "id": "946450a4", "metadata": {}, "source": [ "## D4. One-hot encoding\n" ] }, { "cell_type": "code", "execution_count": null, "id": "7ebab6e3", "metadata": {}, "outputs": [], "source": [ "def one_hot(y, num_classes):\n", " # TODO\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "fc9f4950", "metadata": {}, "outputs": [], "source": [ "fn = get(\"one_hot\")\n", "gb.check(\"D4.1 function callable\", callable(fn), hint=\"Define one_hot(y, num_classes).\")\n", "if callable(fn):\n", " gb.check_array_close(\"D4.2 one-hot\", fn(np.array([2,0,1,2]), 3), np.array([[0,0,1],[1,0,0],[0,1,0],[0,0,1]]))\n" ] }, { "cell_type": "markdown", "id": "9b6d90db", "metadata": {}, "source": [ "## D5. Pairwise Euclidean distances\n" ] }, { "cell_type": "code", "execution_count": null, "id": "4353c475", "metadata": {}, "outputs": [], "source": [ "def pairwise_distances(A, B):\n", " # TODO: return an array with shape (A rows, B rows)\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "9dc33c12", "metadata": {}, "outputs": [], "source": [ "A_D5 = np.array([[0, 0], [1, 0]], dtype=float)\n", "B_D5 = np.array([[0, 1], [1, 1], [2, 0]], dtype=float)\n", "fn = get(\"pairwise_distances\")\n", "gb.check(\"D5.1 function callable\", callable(fn), hint=\"Define pairwise_distances(A, B).\")\n", "if callable(fn):\n", " gb.check_array_close(\"D5.2 distances\", fn(A_D5, B_D5), np.array([[1.0, np.sqrt(2), 2.0], [np.sqrt(2), 1.0, 1.0]]), hint=\"Broadcast A[:,None,:] - B[None,:,:].\")\n" ] }, { "cell_type": "markdown", "id": "df723ac0", "metadata": {}, "source": [ "## D6. Nearest-neighbor labels\n" ] }, { "cell_type": "code", "execution_count": null, "id": "45f8fc0c", "metadata": {}, "outputs": [], "source": [ "def nearest_labels(train_X, train_y, query_X):\n", " # TODO: label of nearest training point for each query row\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "6a9b74e3", "metadata": {}, "outputs": [], "source": [ "train_X_D6 = np.array([[0,0], [1,0], [0,1], [3,3]], dtype=float)\n", "train_y_D6 = np.array([0, 0, 1, 2])\n", "query_X_D6 = np.array([[0.1, 0.9], [2.9, 3.1]], dtype=float)\n", "fn = get(\"nearest_labels\")\n", "gb.check(\"D6.1 function callable\", callable(fn), hint=\"Define nearest_labels(train_X, train_y, query_X).\")\n", "if callable(fn):\n", " gb.check_array_close(\"D6.2 nearest labels\", fn(train_X_D6, train_y_D6, query_X_D6), np.array([1, 2]))\n" ] }, { "cell_type": "markdown", "id": "08ec86e1", "metadata": {}, "source": [ "## D7. KNN majority vote for one query\n" ] }, { "cell_type": "code", "execution_count": null, "id": "f2e9db75", "metadata": {}, "outputs": [], "source": [ "def knn_predict_one(train_X, train_y, x_query, k):\n", " # TODO: break ties by choosing the smaller label\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "e4edeafd", "metadata": {}, "outputs": [], "source": [ "fn = get(\"knn_predict_one\")\n", "gb.check(\"D7.1 function callable\", callable(fn), hint=\"Define knn_predict_one(train_X, train_y, x_query, k).\")\n", "if callable(fn):\n", " gb.check_equal(\"D7.2 k=1\", fn(train_X_D6, train_y_D6, np.array([0.1,0.9]), 1), 1)\n", " gb.check_equal(\"D7.3 k=3 tie chooses smaller label\", fn(train_X_D6, train_y_D6, np.array([0.2,0.2]), 3), 0)\n" ] }, { "cell_type": "markdown", "id": "513e877d", "metadata": {}, "source": [ "## D8. KNN predictions for a batch\n" ] }, { "cell_type": "code", "execution_count": null, "id": "8832f916", "metadata": {}, "outputs": [], "source": [ "def knn_predict_batch(train_X, train_y, query_X, k):\n", " # TODO\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "5428e2dc", "metadata": {}, "outputs": [], "source": [ "fn = get(\"knn_predict_batch\")\n", "gb.check(\"D8.1 function callable\", callable(fn), hint=\"Define knn_predict_batch(train_X, train_y, query_X, k).\")\n", "if callable(fn):\n", " gb.check_array_close(\"D8.2 batch predictions\", fn(train_X_D6, train_y_D6, query_X_D6, 1), np.array([1, 2]))\n" ] }, { "cell_type": "markdown", "id": "49a0f882", "metadata": {}, "source": [ "## D9. Top-k indices\n", "\n", "Return the indices of the `k` smallest distances. If distances tie, keep the lower original index first. This deterministic tie rule avoids mysterious grading differences across sorting algorithms.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "9c4ea841", "metadata": {}, "outputs": [], "source": [ "def top_k_smallest_indices(distances, k):\n", " # TODO: use a stable sort so ties keep original index order.\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "59af475e", "metadata": {}, "outputs": [], "source": [ "fn = get(\"top_k_smallest_indices\")\n", "gb.check(\"D9.1 function callable\", callable(fn), hint=\"Define top_k_smallest_indices(distances, k).\")\n", "if callable(fn):\n", " distances_D9 = np.array([4.2, 1.7, 0.9, 0.9, 5.0])\n", " expected_D9 = np.argsort(distances_D9, kind=\"stable\")[:3]\n", " gb.check_array_close(\"D9.2 top 3 with stable ties\", fn(distances_D9, 3), expected_D9, hint=\"Use np.argsort(distances, kind='stable')[:k].\")\n" ] }, { "cell_type": "markdown", "id": "faab9498", "metadata": {}, "source": [ "## D10. Replace missing values with column means\n" ] }, { "cell_type": "code", "execution_count": null, "id": "3fc626c9", "metadata": {}, "outputs": [], "source": [ "def impute_column_means(X):\n", " # TODO: return a copy with np.nan replaced by that column's mean\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "8d304068", "metadata": {}, "outputs": [], "source": [ "X_D10 = np.array([[1.0, np.nan], [3.0, 10.0], [np.nan, 14.0]])\n", "fn = get(\"impute_column_means\")\n", "gb.check(\"D10.1 function callable\", callable(fn), hint=\"Define impute_column_means(X).\")\n", "if callable(fn):\n", " gb.check_array_close(\"D10.2 imputed\", fn(X_D10), np.array([[1.0, 12.0], [3.0, 10.0], [2.0, 14.0]]))\n" ] }, { "cell_type": "markdown", "id": "913b5f1b", "metadata": {}, "source": [ "## D11. Safe standardization with constant columns\n" ] }, { "cell_type": "code", "execution_count": null, "id": "8e91d407", "metadata": {}, "outputs": [], "source": [ "def safe_standardize(X):\n", " # TODO: return standardized_X, mu, sigma_safe where zero std becomes 1\n", " return None, None, None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "8b92a58c", "metadata": {}, "outputs": [], "source": [ "X_D11 = np.array([[1.0, 5.0], [3.0, 5.0], [5.0, 5.0]])\n", "fn = get(\"safe_standardize\")\n", "gb.check(\"D11.1 function callable\", callable(fn), hint=\"Define safe_standardize(X).\")\n", "if callable(fn):\n", " out, _ = attempt(fn, X_D11)\n", "else:\n", " out = None\n", "if isinstance(out, tuple) and len(out) == 3:\n", " Xstd_D11, mu_D11, sigma_D11 = out\n", "else:\n", " Xstd_D11 = mu_D11 = sigma_D11 = None\n", "gb.check_array_close(\"D11.2 mu\", mu_D11, np.array([3.0, 5.0]))\n", "gb.check_array_close(\"D11.3 sigma safe\", sigma_D11, np.array([np.std([1,3,5]), 1.0]))\n", "gb.check_array_close(\"D11.4 second column becomes zeros\", Xstd_D11[:,1] if Xstd_D11 is not None else None, np.array([0.0, 0.0, 0.0]))\n" ] }, { "cell_type": "markdown", "id": "6501dbce", "metadata": {}, "source": [ "## D12. Broadcasting shape quiz\n" ] }, { "cell_type": "code", "execution_count": null, "id": "44d737d2", "metadata": {}, "outputs": [], "source": [ "X_D12 = np.zeros((5, 3))\n", "row_offsets_D12 = np.arange(5).reshape(5, 1)\n", "col_offsets_D12 = np.arange(3)\n", "\n", "# TODO\n", "D12_shape_row_broadcast = None\n", "D12_shape_col_broadcast = None\n", "D12_shape_both = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "deb262b9", "metadata": {}, "outputs": [], "source": [ "gb.check_equal(\"D12.1 row offsets\", get(\"D12_shape_row_broadcast\"), (5, 3))\n", "gb.check_equal(\"D12.2 col offsets\", get(\"D12_shape_col_broadcast\"), (5, 3))\n", "gb.check_equal(\"D12.3 row + col offsets\", get(\"D12_shape_both\"), (5, 3))\n" ] }, { "cell_type": "markdown", "id": "0b28c39e", "metadata": {}, "source": [ "# Arcade 5 — Plots, Evidence, and Communication\n", "\n", "Plots are not decoration. A good plot makes a claim easier to inspect and easier to argue against.\n", "\n", "Accessibility habit: plots should have axis labels, a title when helpful, and a legend or direct labels. Do not make the meaning depend on colour alone.\n" ] }, { "cell_type": "markdown", "id": "bdf57e0e", "metadata": {}, "source": [ "## E1. Labeled scatter plot\n", "\n", "Create a scatter plot for two classes. Use labels and a legend so the plot can be interpreted without relying on colour alone.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "49f9eb0c", "metadata": {}, "outputs": [], "source": [ "def plot_labeled_points(X, y, title=\"Labeled points\"):\n", " # TODO: create fig, ax; plot class 0 and class 1 points; label axes; return fig, ax\n", " return None, None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "adbe43b6", "metadata": {}, "outputs": [], "source": [ "fn = get(\"plot_labeled_points\")\n", "gb.check(\"E1.1 function callable\", callable(fn), hint=\"Define plot_labeled_points(X, y, title).\")\n", "if callable(fn):\n", " fig_ax, err = attempt(fn, np.array([[0,0], [1,1], [2,0], [2,2]]), np.array([0,0,1,1]), \"Demo\")\n", "else:\n", " fig_ax = None\n", "if isinstance(fig_ax, tuple) and len(fig_ax) == 2:\n", " fig_E1, ax_E1 = fig_ax\n", "else:\n", " fig_E1 = ax_E1 = None\n", "gb.check(\"E1.2 returns axes\", ax_E1 is not None and hasattr(ax_E1, \"set_xlabel\"), hint=\"Return fig, ax from matplotlib.\")\n", "gb.check(\"E1.3 has axis labels\", ax_E1 is not None and ax_E1.get_xlabel() != \"\" and ax_E1.get_ylabel() != \"\", hint=\"Label both axes.\")\n", "gb.check(\"E1.4 has legend\", ax_E1 is not None and ax_E1.get_legend() is not None, hint=\"Add a legend or direct class labels so colour is not the only cue.\")\n", "if fig_E1 is not None:\n", " plt.close(fig_E1)\n" ] }, { "cell_type": "markdown", "id": "02da5548", "metadata": {}, "source": [ "## E2. Accuracy\n" ] }, { "cell_type": "code", "execution_count": null, "id": "66c861c3", "metadata": {}, "outputs": [], "source": [ "def accuracy(y_true, y_pred):\n", " # TODO\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "8b649e51", "metadata": {}, "outputs": [], "source": [ "fn = get(\"accuracy\")\n", "gb.check(\"E2.1 function callable\", callable(fn), hint=\"Define accuracy(y_true, y_pred).\")\n", "if callable(fn):\n", " gb.check_close(\"E2.2 accuracy\", fn(np.array([1,0,1,1]), np.array([1,1,1,0])), 0.5)\n" ] }, { "cell_type": "markdown", "id": "4692fe5a", "metadata": {}, "source": [ "## E3. Confusion matrix from scratch\n" ] }, { "cell_type": "code", "execution_count": null, "id": "dc5954b5", "metadata": {}, "outputs": [], "source": [ "def confusion_matrix_2x2(y_true, y_pred):\n", " # TODO: rows true [0,1], columns pred [0,1]\n", " return None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "33c271c6", "metadata": {}, "outputs": [], "source": [ "fn = get(\"confusion_matrix_2x2\")\n", "gb.check(\"E3.1 function callable\", callable(fn), hint=\"Define confusion_matrix_2x2(y_true, y_pred).\")\n", "if callable(fn):\n", " gb.check_array_close(\"E3.2 confusion matrix\", fn(np.array([0,0,1,1,1]), np.array([0,1,1,0,1])), np.array([[1,1],[1,2]]))\n" ] }, { "cell_type": "markdown", "id": "e573be1e", "metadata": {}, "source": [ "## E4. Anscombe's quartet: same summaries, different pictures\n" ] }, { "cell_type": "code", "execution_count": null, "id": "834ba7bb", "metadata": {}, "outputs": [], "source": [ "anscombe_E4 = {\n", " \"I\": (np.array([10,8,13,9,11,14,6,4,12,7,5], dtype=float), np.array([8.04,6.95,7.58,8.81,8.33,9.96,7.24,4.26,10.84,4.82,5.68])),\n", " \"II\": (np.array([10,8,13,9,11,14,6,4,12,7,5], dtype=float), np.array([9.14,8.14,8.74,8.77,9.26,8.10,6.13,3.10,9.13,7.26,4.74])),\n", " \"III\":(np.array([10,8,13,9,11,14,6,4,12,7,5], dtype=float), np.array([7.46,6.77,12.74,7.11,7.81,8.84,6.08,5.39,8.15,6.42,5.73])),\n", " \"IV\": (np.array([8,8,8,8,8,8,8,19,8,8,8], dtype=float), np.array([6.58,5.76,7.71,8.84,8.47,7.04,5.25,12.50,5.56,7.91,6.89])),\n", "}\n", "\n", "# TODO: dictionary mapping quartet name to correlation coefficient\n", "E4_correlations = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "a62a480f", "metadata": {}, "outputs": [], "source": [ "gb.check(\"E4.1 correlations dict\", isinstance(get(\"E4_correlations\"), dict), hint=\"Return a dictionary.\")\n", "if isinstance(get(\"E4_correlations\"), dict):\n", " for name, corr in get(\"E4_correlations\").items():\n", " gb.check_close(f\"E4.2 correlation {name}\", corr, 0.816, tol=0.01, hint=\"All four are surprisingly similar.\")\n" ] }, { "cell_type": "code", "execution_count": null, "id": "accaea78", "metadata": {}, "outputs": [], "source": [ "# Optional visual: run this to see why summaries are not enough.\n", "# fig, axes = plt.subplots(2, 2, figsize=(8, 6), sharex=True, sharey=True)\n", "# for ax, (name, (x, y)) in zip(axes.ravel(), anscombe_E4.items()):\n", "# ax.scatter(x, y)\n", "# ax.set_title(f\"Anscombe {name}\")\n", "# ax.set_xlabel(\"x\")\n", "# ax.set_ylabel(\"y\")\n", "# plt.tight_layout()\n" ] }, { "cell_type": "markdown", "id": "91e23e89", "metadata": {}, "source": [ "## E5. Pick the best validation setting\n" ] }, { "cell_type": "code", "execution_count": null, "id": "65cd509a", "metadata": {}, "outputs": [], "source": [ "k_values_E5 = np.array([1, 3, 5, 7, 9])\n", "val_acc_E5 = np.array([0.68, 0.76, 0.78, 0.78, 0.74])\n", "\n", "# TODO: choose the smallest k tied for best validation accuracy\n", "E5_best_k = None\n", "E5_best_val_accuracy = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "d03e5b01", "metadata": {}, "outputs": [], "source": [ "gb.check_equal(\"E5.1 best k\", get(\"E5_best_k\"), 5, hint=\"Tie-break by smaller k.\")\n", "gb.check_close(\"E5.2 best validation accuracy\", get(\"E5_best_val_accuracy\"), 0.78)\n" ] }, { "cell_type": "markdown", "id": "62968a58", "metadata": {}, "source": [ "## E6. Validation curve plot\n" ] }, { "cell_type": "code", "execution_count": null, "id": "8ae0b359", "metadata": {}, "outputs": [], "source": [ "def plot_validation_curve(k_values, val_scores):\n", " # TODO: return fig, ax with labels and markers/line\n", " return None, None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "a0229a4b", "metadata": {}, "outputs": [], "source": [ "fn = get(\"plot_validation_curve\")\n", "gb.check(\"E6.1 function callable\", callable(fn), hint=\"Define plot_validation_curve(k_values, val_scores).\")\n", "if callable(fn):\n", " fig_ax, err = attempt(fn, k_values_E5, val_acc_E5)\n", "else:\n", " fig_ax = None\n", "if isinstance(fig_ax, tuple) and len(fig_ax) == 2:\n", " fig_E6, ax_E6 = fig_ax\n", "else:\n", " fig_E6 = ax_E6 = None\n", "gb.check(\"E6.2 returns axes\", ax_E6 is not None and hasattr(ax_E6, \"get_xlabel\"))\n", "gb.check(\"E6.3 labels\", ax_E6 is not None and \"k\" in ax_E6.get_xlabel().lower() and \"accuracy\" in ax_E6.get_ylabel().lower())\n", "if fig_E6 is not None:\n", " plt.close(fig_E6)\n" ] }, { "cell_type": "markdown", "id": "d8698db0", "metadata": {}, "source": [ "## E7. Evidence statement\n" ] }, { "cell_type": "code", "execution_count": null, "id": "a0b74d51", "metadata": {}, "outputs": [], "source": [ "# TODO: 2-4 sentences. Mention the baseline, metric, split, and what would change your mind.\n", "E7_evidence_statement = \"\"\n" ] }, { "cell_type": "code", "execution_count": null, "id": "c24ea70c", "metadata": {}, "outputs": [], "source": [ "gb.check_text(\"E7.1 evidence statement\", get(\"E7_evidence_statement\"), keywords=(\"baseline\", \"metric\", \"test\"), min_len=160)\n" ] }, { "cell_type": "markdown", "id": "9f83e66b", "metadata": {}, "source": [ "## E8. Legal or leaky workflow checks\n", "\n", "For each operation, write either `\"legal\"` or `\"leaky\"`. A legal operation uses only information that would be available at that stage of the workflow.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "22975450", "metadata": {}, "outputs": [], "source": [ "E8_leakage_labels = {\n", " \"fit_scaler_on_training_only\": None,\n", " \"fit_scaler_on_all_rows_before_split\": None,\n", " \"choose_k_on_validation\": None,\n", " \"choose_k_on_test\": None,\n", " \"fill_validation_missing_values_with_training_median\": None,\n", " \"include_future_outcome_as_feature\": None,\n", "}\n" ] }, { "cell_type": "code", "execution_count": null, "id": "5a50129c", "metadata": {}, "outputs": [], "source": [ "expected_E8 = {\n", " \"fit_scaler_on_training_only\": \"legal\",\n", " \"fit_scaler_on_all_rows_before_split\": \"leaky\",\n", " \"choose_k_on_validation\": \"legal\",\n", " \"choose_k_on_test\": \"leaky\",\n", " \"fill_validation_missing_values_with_training_median\": \"legal\",\n", " \"include_future_outcome_as_feature\": \"leaky\",\n", "}\n", "labels_E8 = get(\"E8_leakage_labels\")\n", "gb.check(\"E8.1 labels dictionary\", isinstance(labels_E8, dict), hint=\"Use the provided dictionary and fill each value with 'legal' or 'leaky'.\")\n", "if isinstance(labels_E8, dict):\n", " for key, expected in expected_E8.items():\n", " got = str(labels_E8.get(key, \"\")).strip().lower()\n", " gb.check_equal(f\"E8.2 {key}\", got, expected, hint=\"Ask whether this step learned from validation/test/future information.\")\n" ] }, { "cell_type": "markdown", "id": "242d62ac", "metadata": {}, "source": [ "# Quiz Vault — Short Brainteasers\n", "\n", "These are quick checks. Most answers are one number, a tuple, a boolean, or a short string.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "5373f821", "metadata": {}, "outputs": [], "source": [ "# TODOs\n", "QV1_shape = None # shape of np.zeros((10,4)) @ np.ones(4)\n", "QV2_dot_orthogonal = None # dot([1,0], [0,5])\n", "QV3_cosine_opposite = None # cosine similarity of [1,0] and [-2,0]\n", "QV4_projection_perpendicular_norm = None # norm of projection of [0,3] onto [4,0]\n", "QV5_conditional_denominator = None # denominator for P(true=1 | pred=1) in conf_C2\n", "QV6_posterior_percent = None # C3 posterior as a percentage, rounded to 1 decimal\n", "QV7_iqr_upper_fence = None # upper fence for [1,2,3,4,100]\n", "QV8_argmin_index = None # np.argmin([4.2, 1.7, 0.9, 0.9])\n", "QV9_majority_label = None # majority of [1,1,0,2,1]\n", "QV10_accuracy = None # accuracy of [1,0,1,1] vs [1,1,1,1]\n", "QV11_gradient_steepest_ascent = None # True/False: gradient points steepest uphill locally\n", "QV12_same_mean_enough = None # True/False: same mean proves same distribution\n", "QV13_choose_on_split = None # which split is used to choose settings? \"train\", \"validation\", or \"test\"\n", "QV14_final_once_split = None # which split is reserved for final one-time evaluation?\n", "QV15_evidence_needs_baseline = None # True/False\n" ] }, { "cell_type": "code", "execution_count": null, "id": "6aa58ba7", "metadata": {}, "outputs": [], "source": [ "gb.check_equal(\"QV1 shape\", get(\"QV1_shape\"), (10,))\n", "gb.check_close(\"QV2 orthogonal dot\", get(\"QV2_dot_orthogonal\"), 0.0)\n", "gb.check_close(\"QV3 opposite cosine\", get(\"QV3_cosine_opposite\"), -1.0)\n", "gb.check_close(\"QV4 perpendicular projection norm\", get(\"QV4_projection_perpendicular_norm\"), 0.0)\n", "gb.check_equal(\"QV5 conditional denominator\", get(\"QV5_conditional_denominator\"), 45)\n", "gb.check_close(\"QV6 posterior percent\", get(\"QV6_posterior_percent\"), 9.2, tol=0.05)\n", "gb.check_close(\"QV7 IQR upper fence\", get(\"QV7_iqr_upper_fence\"), 7.0)\n", "gb.check_equal(\"QV8 argmin first tie\", get(\"QV8_argmin_index\"), 2)\n", "gb.check_equal(\"QV9 majority\", get(\"QV9_majority_label\"), 1)\n", "gb.check_close(\"QV10 accuracy\", get(\"QV10_accuracy\"), 0.75)\n", "gb.check_equal(\"QV11 gradient direction\", get(\"QV11_gradient_steepest_ascent\"), True)\n", "gb.check_equal(\"QV12 same mean enough\", get(\"QV12_same_mean_enough\"), False)\n", "gb.check_equal(\"QV13 choose split\", get(\"QV13_choose_on_split\"), \"validation\")\n", "gb.check_equal(\"QV14 final split\", get(\"QV14_final_once_split\"), \"test\")\n", "gb.check_equal(\"QV15 baseline\", get(\"QV15_evidence_needs_baseline\"), True)\n" ] }, { "cell_type": "markdown", "id": "479754a0", "metadata": {}, "source": [ "# Final Boss — Gate Triage Evidence Memo\n", "\n", "This final task combines the readiness skills without gradient descent or any 3151 optimization machinery.\n", "\n", "Scenario: an arcade gate system records two numeric sensor features for each gate check. The target `needs_review` is 1 when the gate should be sent for urgent human review and 0 otherwise. The data are synthetic, but the workflow is real: read the data card, compare against a baseline, standardize legally, choose `k` on validation data, evaluate once on test data, inspect the confusion matrix, and write an evidence memo.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "9df90da5", "metadata": {}, "outputs": [], "source": [ "# Synthetic 2D gate-triage data: two overlapping groups.\n", "rng_boss = np.random.default_rng(SEED)\n", "n0, n1 = 90, 70\n", "X0 = rng_boss.normal(loc=[0.0, 0.0], scale=[1.0, 0.8], size=(n0, 2))\n", "X1 = rng_boss.normal(loc=[1.8, 1.4], scale=[1.0, 0.9], size=(n1, 2))\n", "X_boss = np.vstack([X0, X1])\n", "y_boss = np.array([0]*n0 + [1]*n1)\n", "\n", "boss_feature_names = np.array([\"sensor_drift\", \"vibration_spike\"])\n", "boss_target_name = \"needs_review\"\n", "boss_positive_label = 1\n", "\n", "perm = rng_boss.permutation(len(y_boss))\n", "X_boss = X_boss[perm]\n", "y_boss = y_boss[perm]\n", "\n", "n_train = 96\n", "n_val = 32\n", "X_boss_train, y_boss_train = X_boss[:n_train], y_boss[:n_train]\n", "X_boss_val, y_boss_val = X_boss[n_train:n_train+n_val], y_boss[n_train:n_train+n_val]\n", "X_boss_test, y_boss_test = X_boss[n_train+n_val:], y_boss[n_train+n_val:]\n", "\n", "print(\"feature names:\", boss_feature_names)\n", "print(\"target:\", boss_target_name, \"where 1 means urgent review\")\n", "print(\"train/val/test shapes:\", X_boss_train.shape, X_boss_val.shape, X_boss_test.shape)\n", "print(\"train class counts [0, 1]:\", np.bincount(y_boss_train))\n" ] }, { "cell_type": "markdown", "id": "590c9889", "metadata": {}, "source": [ "## F0. Data card and baseline\n", "\n", "Before using KNN, write down the basic data card and a majority-class baseline. A model score is not meaningful until it has something simple to beat.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "17e8e16b", "metadata": {}, "outputs": [], "source": [ "# TODO: answer from the data above.\n", "F0_n_rows = None\n", "F0_n_features = None\n", "F0_target_name = \"\"\n", "F0_baseline_label = None # majority class in the training split\n", "F0_baseline_test_accuracy = None # accuracy if every test row gets the baseline label\n", "F0_possible_harm = \"\" # one sentence\n", "F0_most_costly_error = \"\" # one sentence, e.g. false positive or false negative and why\n" ] }, { "cell_type": "code", "execution_count": null, "id": "c14d5ee8", "metadata": {}, "outputs": [], "source": [ "_boss_baseline_label = int(np.argmax(np.bincount(y_boss_train)))\n", "_boss_baseline_test_accuracy = float(np.mean(y_boss_test == _boss_baseline_label))\n", "\n", "gb.check_equal(\"F0.1 row count\", get(\"F0_n_rows\"), len(y_boss))\n", "gb.check_equal(\"F0.2 feature count\", get(\"F0_n_features\"), X_boss.shape[1])\n", "_target_text = str(get(\"F0_target_name\")).strip().lower().replace(\" \", \"_\")\n", "gb.check(\"F0.3 target name\", \"needs_review\" in _target_text, hint=\"The target is named needs_review.\")\n", "gb.check_equal(\"F0.4 majority baseline label\", get(\"F0_baseline_label\"), _boss_baseline_label, hint=\"Use the most common label in y_boss_train.\")\n", "gb.check_close(\"F0.5 baseline test accuracy\", get(\"F0_baseline_test_accuracy\"), _boss_baseline_test_accuracy, hint=\"Predict the baseline label for every test row.\")\n", "gb.check_text(\"F0.6 possible harm\", get(\"F0_possible_harm\"), min_len=40, hint=\"Name one possible harm of a wrong triage decision.\")\n", "gb.check_text(\"F0.7 costly error\", get(\"F0_most_costly_error\"), min_len=40, hint=\"State whether false positives or false negatives worry you more, and why.\")\n" ] }, { "cell_type": "markdown", "id": "6f153d42", "metadata": {}, "source": [ "## F1. Standardize the boss data\n", "\n", "Use training statistics only. Validation and test rows may be transformed by those statistics, but they must not help compute them.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "ad8b516b", "metadata": {}, "outputs": [], "source": [ "# TODO: use fit_standardize from Arcade 1, or reproduce the same train-only logic.\n", "X_boss_train_std = None\n", "X_boss_val_std = None\n", "X_boss_test_std = None\n", "boss_mu = None\n", "boss_sigma = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "e75d486e", "metadata": {}, "outputs": [], "source": [ "_ref_boss_mu = X_boss_train.mean(axis=0)\n", "_ref_boss_sigma = X_boss_train.std(axis=0)\n", "gb.check_array_close(\"F1.1 train mean standardized\", np.mean(get(\"X_boss_train_std\"), axis=0) if get(\"X_boss_train_std\") is not None else None, np.array([0.0, 0.0]), tol=1e-10)\n", "gb.check_array_close(\"F1.2 train std standardized\", np.std(get(\"X_boss_train_std\"), axis=0) if get(\"X_boss_train_std\") is not None else None, np.array([1.0, 1.0]), tol=1e-10)\n", "gb.check_array_close(\"F1.3 boss mu\", get(\"boss_mu\"), _ref_boss_mu)\n", "gb.check_array_close(\"F1.4 boss sigma\", get(\"boss_sigma\"), _ref_boss_sigma)\n", "gb.check_array_close(\"F1.5 validation standardized\", get(\"X_boss_val_std\"), (X_boss_val - _ref_boss_mu)/_ref_boss_sigma)\n", "gb.check_array_close(\"F1.6 test standardized\", get(\"X_boss_test_std\"), (X_boss_test - _ref_boss_mu)/_ref_boss_sigma)\n" ] }, { "cell_type": "markdown", "id": "069c39e4", "metadata": {}, "source": [ "## F2. Evaluate several values of k on validation data\n", "\n", "Use validation accuracy to compare candidate values of `k`. Do not look at the test set while choosing.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "59620efa", "metadata": {}, "outputs": [], "source": [ "k_values_F2 = np.array([1, 3, 5, 7, 9, 11])\n", "\n", "# TODO: array of validation accuracies, one per k\n", "F2_val_accuracies = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "9035c5a2", "metadata": {}, "outputs": [], "source": [ "def _reference_pairwise(A, B):\n", " A = np.asarray(A, dtype=float)\n", " B = np.asarray(B, dtype=float)\n", " return np.sqrt(((A[:, None, :] - B[None, :, :])**2).sum(axis=2))\n", "\n", "def _reference_knn_batch(train_X, train_y, query_X, k):\n", " d = _reference_pairwise(query_X, train_X)\n", " idx = np.argsort(d, axis=1, kind=\"stable\")[:, :k]\n", " preds = []\n", " for row in idx:\n", " counts = np.bincount(train_y[row])\n", " preds.append(int(np.argmax(counts)))\n", " return np.array(preds)\n", "\n", "_ref_F2 = np.array([\n", " np.mean(y_boss_val == _reference_knn_batch((X_boss_train - X_boss_train.mean(axis=0))/X_boss_train.std(axis=0), y_boss_train, (X_boss_val - X_boss_train.mean(axis=0))/X_boss_train.std(axis=0), int(k)))\n", " for k in k_values_F2\n", "])\n", "gb.check_array_close(\"F2.1 validation accuracies\", get(\"F2_val_accuracies\"), _ref_F2, hint=\"For each k, predict validation labels and compute accuracy.\")\n" ] }, { "cell_type": "markdown", "id": "5a48e1fa", "metadata": {}, "source": [ "## F3. Choose the best k using validation data\n", "\n", "If several values tie, choose the smallest tied `k`. This is a fixed tie-break rule, not a test-set decision.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "25c29685", "metadata": {}, "outputs": [], "source": [ "# TODO: choose the smallest k tied for best validation accuracy.\n", "F3_best_k = None\n", "F3_best_val_accuracy = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "84b13831", "metadata": {}, "outputs": [], "source": [ "_ref_best_idx = np.flatnonzero(_ref_F2 == _ref_F2.max())[0]\n", "gb.check_equal(\"F3.1 best k\", get(\"F3_best_k\"), int(k_values_F2[_ref_best_idx]))\n", "gb.check_close(\"F3.2 best validation accuracy\", get(\"F3_best_val_accuracy\"), float(_ref_F2[_ref_best_idx]))\n" ] }, { "cell_type": "markdown", "id": "5cd8262e", "metadata": {}, "source": [ "## F4. Evaluate once on the test set\n", "\n", "Now freeze the choice of `k` and evaluate once. Report accuracy, a confusion matrix, and whether KNN beat the majority-class baseline.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "89fba03a", "metadata": {}, "outputs": [], "source": [ "# TODO: use F3_best_k. Do not choose k on the test set.\n", "F4_test_predictions = None\n", "F4_test_accuracy = None\n", "F4_confusion_matrix = None # rows true [0,1], columns predicted [0,1]\n", "F4_knn_beats_baseline = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "6cc5c063", "metadata": {}, "outputs": [], "source": [ "_ref_train_std = (X_boss_train - X_boss_train.mean(axis=0))/X_boss_train.std(axis=0)\n", "_ref_test_std = (X_boss_test - X_boss_train.mean(axis=0))/X_boss_train.std(axis=0)\n", "_ref_test_pred = _reference_knn_batch(_ref_train_std, y_boss_train, _ref_test_std, int(k_values_F2[_ref_best_idx]))\n", "_ref_test_acc = float(np.mean(y_boss_test == _ref_test_pred))\n", "_ref_conf_F4 = np.zeros((2, 2), dtype=int)\n", "for true_label, pred_label in zip(y_boss_test, _ref_test_pred):\n", " _ref_conf_F4[int(true_label), int(pred_label)] += 1\n", "_ref_baseline_label_F4 = int(np.argmax(np.bincount(y_boss_train)))\n", "_ref_baseline_test_acc_F4 = float(np.mean(y_boss_test == _ref_baseline_label_F4))\n", "\n", "gb.check_array_close(\"F4.1 test predictions\", get(\"F4_test_predictions\"), _ref_test_pred)\n", "gb.check_close(\"F4.2 test accuracy\", get(\"F4_test_accuracy\"), _ref_test_acc)\n", "gb.check_array_close(\"F4.3 test confusion matrix\", get(\"F4_confusion_matrix\"), _ref_conf_F4, hint=\"Rows are true [0,1]; columns are predicted [0,1].\")\n", "gb.check_equal(\"F4.4 beats baseline\", get(\"F4_knn_beats_baseline\"), bool(_ref_test_acc > _ref_baseline_test_acc_F4), hint=\"Compare KNN test accuracy with the majority-class baseline test accuracy.\")\n" ] }, { "cell_type": "markdown", "id": "0b4555c2", "metadata": {}, "source": [ "## F5. Plot validation accuracy versus k\n", "\n", "The plot is part of the evidence trail: it shows why the chosen `k` was reasonable before the test set was used.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "856d6e63", "metadata": {}, "outputs": [], "source": [ "# TODO: use plot_validation_curve and store fig, ax.\n", "F5_fig = None\n", "F5_ax = None\n" ] }, { "cell_type": "code", "execution_count": null, "id": "51c414a4", "metadata": {}, "outputs": [], "source": [ "gb.check(\"F5.1 has axes\", get(\"F5_ax\") is not None and hasattr(get(\"F5_ax\"), \"get_xlabel\"), hint=\"Store the returned ax in F5_ax.\")\n", "gb.check(\"F5.2 labeled x-axis\", get(\"F5_ax\") is not None and \"k\" in get(\"F5_ax\").get_xlabel().lower())\n", "gb.check(\"F5.3 labeled y-axis\", get(\"F5_ax\") is not None and \"accuracy\" in get(\"F5_ax\").get_ylabel().lower())\n", "if get(\"F5_fig\") is not None:\n", " plt.close(get(\"F5_fig\"))\n" ] }, { "cell_type": "markdown", "id": "bdf393a8", "metadata": {}, "source": [ "## F6. Final evidence memo\n", "\n", "Write a concise evidence memo. The goal is not to sound impressive; the goal is to make the claim inspectable.\n", "\n", "Use this structure:\n", "\n", "```text\n", "Question:\n", "Rows / features / target:\n", "Baseline:\n", "Model and validation choice:\n", "Preprocessing learned from:\n", "Final test result:\n", "Confusion matrix / important errors:\n", "Leakage checks:\n", "Main uncertainty:\n", "Decision:\n", "What would change my mind:\n", "```\n" ] }, { "cell_type": "code", "execution_count": null, "id": "01410826", "metadata": {}, "outputs": [], "source": [ "# TODO: 1-2 short paragraphs using the evidence memo structure above.\n", "F6_evidence_memo = \"\"\"\n", "\n", "\"\"\"\n" ] }, { "cell_type": "code", "execution_count": null, "id": "3b5bc7ad", "metadata": {}, "outputs": [], "source": [ "memo = get(\"F6_evidence_memo\")\n", "gb.check_text(\"F6.1 evidence memo length\", memo, min_len=320, hint=\"Write a concise but complete memo, not just a score.\")\n", "for keyword in [\"baseline\", \"validation\", \"test\", \"confusion\", \"leakage\", \"uncertainty\"]:\n", " gb.check(f\"F6.2 mentions {keyword}\", isinstance(memo, str) and keyword in memo.lower(), hint=f\"Mention {keyword} explicitly.\")\n", "gb.check(\"F6.3 mentions k\", isinstance(memo, str) and \"k\" in memo.lower(), hint=\"State which k was chosen and how.\")\n", "gb.check(\"F6.4 change-my-mind clause\", isinstance(memo, str) and any(phrase in memo.lower() for phrase in [\"change my mind\", \"trust\", \"distrust\", \"would change\"]), hint=\"Say what evidence would change your mind.\")\n" ] }, { "cell_type": "markdown", "id": "6d4a76a0", "metadata": {}, "source": [ "# Summary\n", "\n", "Run this after completing the notebook. Because the Gradebook stores the latest result for each check, repaired checks replace earlier failures.\n" ] }, { "cell_type": "code", "execution_count": null, "id": "fed8a6c8", "metadata": {}, "outputs": [], "source": [ "gb.summary()\n" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "name": "python", "pygments_lexer": "ipython3" } }, "nbformat": 4, "nbformat_minor": 5 }