"""Chapter 4 (section 4.3 and exercise 4.2): gradient descent on a one-parameter loss curve. The loss curve is L(w) = (w - 3)^2, whose slope at w is 2 * (w - 3). Each step moves w by -(learning rate) * (slope). The script prints the first steps for three learning rates. """ def loss(w): return (w - 3) ** 2 def slope(w): return 2 * (w - 3) def run(lr, w0=0.0, steps=5): print(f"learning rate = {lr}") w = w0 print(f" step 0: w = {w:.4f}, slope = {slope(w):+.4f}, loss = {loss(w):.4f}") for k in range(1, steps + 1): w = w - lr * slope(w) print(f" step {k}: w = {w:.4f}, slope = {slope(w):+.4f}, loss = {loss(w):.4f}") print() if __name__ == "__main__": for lr in (0.1, 1.1): run(lr) # how many steps with lr = 0.1 until w is within 0.01 of the minimum at 3 w, k = 0.0, 0 while abs(w - 3) >= 0.01: w = w - 0.1 * slope(w) k += 1 print(f"lr = 0.1: steps until |w - 3| < 0.01: {k} (w = {w:.4f})")