import numpy as np, pandas as pd, matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor, plot_tree
from sklearn.metrics import roc_auc_score, accuracy_score, mean_squared_error
import cart
BLUE="#2b6cb0"; RED="#c53030"; GREEN="#2f855a"; ORANGE="#dd6b20"
d = pd.read_csv("credit_default.csv"); feat = [c for c in d.columns if c!="default"]
X = d[feat].to_numpy(float); y = d["default"].to_numpy(int)
Xtr,Xte,ytr,yte = train_test_split(X,y,test_size=0.3,random_state=0,stratify=y)
print(f"credit default: {X.shape[0]:,} clients, {X.shape[1]} features; default rate {y.mean():.1%}")
print(f"train {Xtr.shape[0]:,} / test {Xte.shape[0]:,}")

credit default: 30,000 clients, 23 features; default rate 22.1%
train 21,000 / test 9,000

print(f"clients: {len(d):,}   features: {len(feat)}   overall default rate: {y.mean():.1%}")
fig,ax=plt.subplots(1,2,figsize=(14,4.2))
# default rate by most-recent repayment status
g=d.groupby("PAY_1")["default"].agg(["mean","size"]); g=g[g["size"]>=50]
ax[0].bar(g.index, g["mean"], color=[RED if k>=1 else BLUE for k in g.index])
ax[0].axhline(y.mean(),color="k",ls="--",lw=1,label=f"overall {y.mean():.0%}")
ax[0].set_xlabel("PAY_1  (months in arrears last month; ≤0 = paid/revolving)"); ax[0].set_ylabel("default rate")
ax[0].set_title("Recent delinquency is the dominant risk signal"); ax[0].legend(frameon=False)
# default rate by credit-limit quartile
d["_limq"]=pd.qcut(d["LIMIT_BAL"],4,labels=["Q1 low","Q2","Q3","Q4 high"])
gl=d.groupby("_limq",observed=True)["default"].mean()
ax[1].bar(range(4), gl.values, color=ORANGE); ax[1].set_xticks(range(4)); ax[1].set_xticklabels(gl.index)
ax[1].axhline(y.mean(),color="k",ls="--",lw=1); ax[1].set_xlabel("credit-limit quartile"); ax[1].set_ylabel("default rate")
ax[1].set_title("Lower credit limits default more")
plt.tight_layout(); plt.show()
r0=d.loc[d.PAY_1<=0,"default"].mean(); r2=d.loc[d.PAY_1>=2,"default"].mean()
print(f"CONCLUSION (raw data): a client who paid on time last month defaults {r0:.0%} of the time; one already 2+ months behind defaults {r2:.0%} -- a {r2/r0:.1f}x jump.")
print("Recent repayment behaviour dwarfs demographics as a risk signal. The tree should seize on PAY_1 first.")

clients: 30,000   features: 23   overall default rate: 22.1%

CONCLUSION (raw data): a client who paid on time last month defaults 14% of the time; one already 2+ months behind defaults 70% -- a 5.0x jump.
Recent repayment behaviour dwarfs demographics as a risk signal. The tree should seize on PAY_1 first.

# evaluate impurity decrease across thresholds of the most predictive raw feature (PAY_1 = latest repayment status)
j = feat.index("PAY_1"); xs = Xtr[:,j]
tree1 = cart.DecisionTree("gini", max_depth=1).fit(Xtr,ytr)
root = tree1.root
def gini(yy):
    p=np.bincount(yy,minlength=2)/len(yy); return 1-(p*p).sum()
base = gini(ytr); vals=np.unique(xs); dec=[]
for thr in vals[:-1]:
    m=xs<=thr; nl=m.sum(); nr=len(xs)-nl
    if nl<50 or nr<50: dec.append(np.nan); continue
    dec.append(base - (nl*gini(ytr[m]) + nr*gini(ytr[~m]))/len(xs))
fig,ax=plt.subplots(figsize=(7.5,4))
ax.plot(vals[:-1], dec, "o-", color=BLUE, ms=4)
ax.axvline(root.thr, color=RED, ls="--", label=f"CART best split: PAY_1 ≤ {root.thr:.1f}")
ax.set_xlabel("PAY_1 threshold (months delayed)"); ax.set_ylabel("Gini impurity decrease  Δi")
ax.set_title("The greedy split search on one feature"); ax.legend(frameon=False)
plt.tight_layout(); plt.show()
print(f"Best root split chosen by CART: {feat[root.feat]} ≤ {root.thr:.2f}  (Gini decrease {base-(root.left.imp*root.left.n+root.right.imp*root.right.n)/root.n:.4f})")
print("PAY_1 -- whether the client was already behind last month -- is the single most informative question, as intuition expects.")

Best root split chosen by CART: PAY_1 ≤ 1.50  (Gini decrease 0.0526)
PAY_1 -- whether the client was already behind last month -- is the single most informative question, as intuition expects.

tree3 = cart.DecisionTree("gini", max_depth=3).fit(Xtr,ytr)
def show(node, depth=0, tag="root"):
    ind="   "*depth
    if node.feat is None:
        k=int(np.argmax(node.value)); print(f"{ind}[{tag}] -> predict {k} (p_default={node.value[1]:.2f}, n={node.n})")
    else:
        print(f"{ind}[{tag}] {feat[node.feat]} <= {node.thr:.1f} ?   gini={node.imp:.3f} n={node.n}")
        show(node.left, depth+1, "yes"); show(node.right, depth+1, "no")
show(tree3.root)
sk3 = DecisionTreeClassifier(max_depth=3, random_state=0).fit(Xtr,ytr)
fig,ax=plt.subplots(figsize=(15,6))
plot_tree(sk3, feature_names=feat, class_names=["no","default"], filled=True, impurity=True, proportion=True, rounded=True, fontsize=8, ax=ax)
ax.set_title("The same depth-3 tree, drawn by scikit-learn's plot_tree"); plt.tight_layout(); plt.show()

[root] PAY_1 <= 1.5 ?   gini=0.345 n=21000
   [yes] PAY_2 <= 1.5 ?   gini=0.276 n=18792
      [yes] PAY_AMT3 <= 678.5 ?   gini=0.244 n=17183
         [yes] -> predict 0 (p_default=0.22, n=4731)
         [no] -> predict 0 (p_default=0.11, n=12452)
      [no] PAY_6 <= 1.0 ?   gini=0.486 n=1609
         [yes] -> predict 0 (p_default=0.38, n=1153)
         [no] -> predict 1 (p_default=0.52, n=456)
   [no] PAY_6 <= 1.0 ?   gini=0.425 n=2208
      [yes] BILL_AMT1 <= 2207.5 ?   gini=0.457 n=1330
         [yes] -> predict 0 (p_default=0.43, n=124)
         [no] -> predict 1 (p_default=0.67, n=1206)
      [no] PAY_AMT1 <= 15176.5 ?   gini=0.358 n=878
         [yes] -> predict 1 (p_default=0.77, n=871)
         [no] -> predict 0 (p_default=0.14, n=7)

rows=[]
for depth in [2,3,5,8]:
    ms = cart.DecisionTree("gini", max_depth=depth).fit(Xtr,ytr)
    sk = DecisionTreeClassifier(max_depth=depth, random_state=0).fit(Xtr,ytr)
    rows.append([depth,
        accuracy_score(yte,ms.predict(Xte)), accuracy_score(yte,sk.predict(Xte)),
        roc_auc_score(yte,ms.predict_proba(Xte)[:,1]), roc_auc_score(yte,sk.predict_proba(Xte)[:,1]),
        ms.n_leaves(), sk.get_n_leaves()])
tab=pd.DataFrame(rows,columns=["depth","acc_scratch","acc_sklearn","auc_scratch","auc_sklearn","leaves_scratch","leaves_sklearn"])
print(tab.round(4).to_string(index=False))
ident = [(d, (cart.DecisionTree("gini", max_depth=d).fit(Xtr,ytr).predict(Xte) ==
              DecisionTreeClassifier(max_depth=d, random_state=0).fit(Xtr,ytr).predict(Xte)).mean())
         for d in (2,3,5,8)]
print("\nAgreement, stated exactly: " + "  ".join(f"depth {d}: {f:.2%} identical labels" for d,f in ident))
print("The two agree exactly while the tree is shallow and drift apart gracefully as it deepens.")
print("The divergence")
print("that appears with depth is TIE-BREAKING, not a difference of algorithm: when two splits give exactly the")
print("same impurity decrease, scikit-learn scans features in a randomised order and this implementation scans")
print("them in index order, so the tie falls the other way and the subtree below it differs. The metrics still")
print("agree to 3-4 decimals, which is the useful test -- but 'identical' would be the wrong word for deep trees.")

 depth  acc_scratch  acc_sklearn  auc_scratch  auc_sklearn  leaves_scratch  leaves_sklearn
     2       0.8194       0.8194       0.6898       0.6898               4               4
     3       0.8214       0.8214       0.7259       0.7259               8               8
     5       0.8194       0.8193       0.7463       0.7462              31              31
     8       0.8116       0.8122       0.7362       0.7365             152             151

Agreement, stated exactly: depth 2: 100.00% identical labels  depth 3: 100.00% identical labels  depth 5: 99.99% identical labels  depth 8: 99.31% identical labels
The two agree exactly while the tree is shallow and drift apart gracefully as it deepens.
The divergence
that appears with depth is TIE-BREAKING, not a difference of algorithm: when two splits give exactly the
same impurity decrease, scikit-learn scans features in a randomised order and this implementation scans
them in index order, so the tie falls the other way and the subtree below it differs. The metrics still
agree to 3-4 decimals, which is the useful test -- but 'identical' would be the wrong word for deep trees.

depths=range(1,21); tr_auc=[]; te_auc=[]
for dep in depths:
    sk=DecisionTreeClassifier(max_depth=dep,random_state=0).fit(Xtr,ytr)
    tr_auc.append(roc_auc_score(ytr,sk.predict_proba(Xtr)[:,1]))
    te_auc.append(roc_auc_score(yte,sk.predict_proba(Xte)[:,1]))
fig,ax=plt.subplots(figsize=(7.5,4))
ax.plot(list(depths),tr_auc,"o-",color=BLUE,label="train AUC")
ax.plot(list(depths),te_auc,"s-",color=RED,label="test AUC")
best=list(depths)[int(np.argmax(te_auc))]; ax.axvline(best,color=GREEN,ls=":",label=f"test peak ~depth {best}")
ax.set_xlabel("max_depth"); ax.set_ylabel("ROC-AUC"); ax.set_title("One tree overfits: train AUC → 1, test AUC turns over")
ax.legend(frameon=False); plt.tight_layout(); plt.show()
print(f"Train AUC climbs toward 1.0 while test AUC peaks near depth {best} then falls -- textbook high-variance overfitting.")

Train AUC climbs toward 1.0 while test AUC peaks near depth 6 then falls -- textbook high-variance overfitting.

rng=np.random.default_rng(0); n=len(Xtr); Bn=25
Pm=np.zeros((Bn,len(Xte)))
for b in range(Bn):
    idx=rng.integers(0,n,n)                                              # bootstrap resample
    Pm[b]=DecisionTreeClassifier(min_samples_leaf=5,random_state=0).fit(Xtr[idx],ytr[idx]).predict_proba(Xte)[:,1]
sd=Pm.std(0); aucs=[roc_auc_score(yte,Pm[b]) for b in range(Bn)]; avg=roc_auc_score(yte,Pm.mean(0))
fig,ax=plt.subplots(1,2,figsize=(13,4))
ax[0].hist(sd,bins=40,color=ORANGE); ax[0].axvline(sd.mean(),color=RED,lw=2,label=f"mean {sd.mean():.2f}")
ax[0].set_xlabel("SD of predicted P(default) across 25 deep trees"); ax[0].set_ylabel("test clients"); ax[0].set_title("Same client, very different predictions"); ax[0].legend(frameon=False)
ax[1].hist(aucs,bins=10,color=BLUE,alpha=.85); ax[1].axvline(np.mean(aucs),color=RED,lw=2,label=f"single tree, mean {np.mean(aucs):.3f}")
ax[1].axvline(avg,color=GREEN,lw=2,ls="--",label=f"25 trees averaged {avg:.3f}"); ax[1].set_xlabel("test AUC"); ax[1].set_title("Averaging beats any single tree"); ax[1].legend(frameon=False,fontsize=8)
plt.tight_layout(); plt.show()
print(f"A single deep tree pins a given client's default probability only to SD ~{sd.mean():.2f} across resamples; its test AUC is {np.mean(aucs):.3f} ± {np.std(aucs):.3f}.")
print(f"AVERAGING the 25 trees lifts AUC to {avg:.3f} -- the variance cancels while the signal adds. That is BAGGING, and the Random Forest refines it (next notebook).")

A single deep tree pins a given client's default probability only to SD ~0.26 across resamples; its test AUC is 0.651 ± 0.006.
AVERAGING the 25 trees lifts AUC to 0.758 -- the variance cancels while the signal adds. That is BAGGING, and the Random Forest refines it (next notebook).

full = cart.DecisionTree("gini", max_depth=None, min_samples_leaf=5).fit(Xtr,ytr)
alphas = full.cost_complexity_pruning_path()
sk_alphas = DecisionTreeClassifier(random_state=0,min_samples_leaf=5).fit(Xtr,ytr).cost_complexity_pruning_path(Xtr,ytr).ccp_alphas
print(f"weakest-link path: {len(alphas)} alphas (from-scratch) vs {len(sk_alphas)} (sklearn); ranges "
      f"[{alphas.min():.1e},{alphas.max():.1e}] vs [{sk_alphas.min():.1e},{sk_alphas.max():.1e}]")
grid = np.unique(np.concatenate([[0], np.geomspace(1e-5, alphas.max(), 25)]))
leaves=[]; tr=[]; te=[]; cvv=[]
for a in grid:
    m=cart.DecisionTree("gini",min_samples_leaf=5,ccp_alpha=a).fit(Xtr,ytr)
    leaves.append(m.n_leaves()); tr.append(roc_auc_score(ytr,m.predict_proba(Xtr)[:,1])); te.append(roc_auc_score(yte,m.predict_proba(Xte)[:,1]))
    sk=DecisionTreeClassifier(random_state=0,min_samples_leaf=5,ccp_alpha=a)
    cvv.append(cross_val_score(sk,Xtr,ytr,cv=4,scoring="roc_auc").mean())
astar=grid[int(np.argmax(cvv))]
fig,ax=plt.subplots(1,2,figsize=(14,4))
ax[0].plot(grid,leaves,"o-",color=ORANGE); ax[0].set_xscale("log"); ax[0].set_xlabel("ccp_alpha"); ax[0].set_ylabel("number of leaves"); ax[0].set_title("Pruning shrinks the tree as α grows")
ax[1].plot(grid,tr,"o-",color=BLUE,label="train AUC"); ax[1].plot(grid,te,"s-",color=RED,label="test AUC"); ax[1].plot(grid,cvv,"^--",color=GREEN,label="4-fold CV AUC")
ax[1].axvline(astar,color="k",ls=":",label=f"CV-best α={astar:.1e}"); ax[1].set_xscale("log"); ax[1].set_xlabel("ccp_alpha"); ax[1].set_ylabel("ROC-AUC"); ax[1].set_title("CV picks the pruning strength"); ax[1].legend(frameon=False,fontsize=8)
plt.tight_layout(); plt.show()
mstar=cart.DecisionTree("gini",min_samples_leaf=5,ccp_alpha=astar).fit(Xtr,ytr)
print(f"CV-optimal α prunes {full.n_leaves()} -> {mstar.n_leaves()} leaves; test AUC {roc_auc_score(yte,full.predict_proba(Xte)[:,1]):.4f} -> {roc_auc_score(yte,mstar.predict_proba(Xte)[:,1]):.4f}")
print("Pruning a smaller tree GENERALISES BETTER: fewer leaves, higher test AUC. This is the bias-variance trade-off, tuned.")
print(f"The two paths differ slightly in length ({len(alphas)} against {len(sk_alphas)}) while spanning the identical alpha")
print("range. That is inherited, not a different pruning rule: the two fully-grown trees already differ by a")
print("handful of leaves through the tie-breaking above, and the weakest-link sequence is computed from them.")
shared = np.quantile(sk_alphas, [0.5, 0.9, 0.99])
sizes = [(a, cart.DecisionTree("gini", max_depth=None, min_samples_leaf=5, ccp_alpha=a).fit(Xtr,ytr).n_leaves(),
          DecisionTreeClassifier(random_state=0, min_samples_leaf=5, ccp_alpha=a).fit(Xtr,ytr).get_n_leaves())
         for a in shared]
print("Pruned at a shared alpha: " + ",  ".join(f"a={a:.1e} -> {u} vs {v} leaves" for a,u,v in sizes))
print("The gap closes as alpha grows: a percent or two apart while the tree is still large and the accumulated")
print("tie-breaks still matter, and exact by the time alpha has pruned it to the size cross-validation selects.")

weakest-link path: 895 alphas (from-scratch) vs 920 (sklearn); ranges [0.0e+00,5.3e-02] vs [0.0e+00,5.3e-02]

CV-optimal α prunes 1804 -> 7 leaves; test AUC 0.6535 -> 0.7443
Pruning a smaller tree GENERALISES BETTER: fewer leaves, higher test AUC. This is the bias-variance trade-off, tuned.
The two paths differ slightly in length (895 against 920) while spanning the identical alpha
range. That is inherited, not a different pruning rule: the two fully-grown trees already differ by a
handful of leaves through the tie-breaking above, and the weakest-link sequence is computed from them.

Pruned at a shared alpha: a=5.7e-05 -> 1148 vs 1131 leaves,  a=1.5e-04 -> 215 vs 204 leaves,  a=5.1e-04 -> 11 vs 11 leaves
The gap closes as alpha grows: a percent or two apart while the tree is still large and the accumulated
tie-breaks still matter, and exact by the time alpha has pruned it to the size cross-validation selects.

h = pd.read_csv("cali_housing.csv"); hf=[c for c in h.columns if c!="MedHouseVal"]
Xh=h[hf].to_numpy(float); yh=h["MedHouseVal"].to_numpy(float)
Xhtr,Xhte,yhtr,yhte=train_test_split(Xh,yh,test_size=0.3,random_state=0)
# 1-D illustration on median income
xi=Xhtr[:,hf.index("MedInc")]; order=np.argsort(xi); grid=np.linspace(xi.min(),xi.max(),400)
fig,ax=plt.subplots(figsize=(8,4.5))
ax.scatter(xi, yhtr, s=4, alpha=0.12, color="grey")
for dep,col in zip([2,5,None],[GREEN,BLUE,RED]):
    m=cart.DecisionTree("mse",max_depth=dep).fit(xi[:,None],yhtr)
    ax.plot(grid, m.predict(grid[:,None]), color=col, lw=2, label=f"depth {dep if dep else 'full'}")
ax.set_xlabel("median income (block group)"); ax.set_ylabel("median house value ($100k)")
ax.set_title("Regression tree = step function; deeper = finer, eventually noisy"); ax.legend(frameon=False)
plt.tight_layout(); plt.show()
# full multivariate check vs sklearn
for dep in [4,8]:
    ms=cart.DecisionTree("mse",max_depth=dep).fit(Xhtr,yhtr); sk=DecisionTreeRegressor(max_depth=dep,random_state=0).fit(Xhtr,yhtr)
    print(f"depth {dep}: RMSE from-scratch {mean_squared_error(yhte,ms.predict(Xhte))**.5:.4f} | sklearn {mean_squared_error(yhte,sk.predict(Xhte))**.5:.4f}")

depth 4: RMSE from-scratch 0.7705 | sklearn 0.7705
depth 8: RMSE from-scratch 0.6654 | sklearn 0.6658

mstar_imp = mstar.feature_importances_
imp = pd.Series(mstar_imp, index=feat).sort_values(ascending=False).head(8)
fig,ax=plt.subplots(1,2,figsize=(14,4.6))
imp[::-1].plot.barh(ax=ax[0], color=BLUE); ax[0].set_title("Gini feature importances (pruned tree)"); ax[0].set_xlabel("impurity decrease (normalised)")
# decision surface on PAY_1 x LIMIT_BAL
a_,b_=feat.index("PAY_1"), feat.index("LIMIT_BAL")
surf=DecisionTreeClassifier(max_depth=4,random_state=0).fit(Xtr[:,[a_,b_]],ytr)
xx,yy=np.meshgrid(np.linspace(Xtr[:,a_].min(),Xtr[:,a_].max(),200), np.linspace(0,Xtr[:,b_].max(),200))
zz=surf.predict_proba(np.c_[xx.ravel(),yy.ravel()])[:,1].reshape(xx.shape)
cs=ax[1].contourf(xx,yy,zz,levels=12,cmap="RdBu_r",alpha=0.8); plt.colorbar(cs,ax=ax[1],shrink=0.7,label="P(default)")
ax[1].set_xlabel("PAY_1 (repayment status)"); ax[1].set_ylabel("LIMIT_BAL (credit limit)"); ax[1].set_title("Axis-aligned decision surface (2 features)")
plt.tight_layout(); plt.show()
print("Recent repayment status (PAY_1) dominates; the surface shows the rectangular, axis-aligned regions a tree produces.")

Recent repayment status (PAY_1) dominates; the surface shows the rectangular, axis-aligned regions a tree produces.

from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import make_pipeline
from sklearn.ensemble import RandomForestClassifier
import statsmodels.api as sm
logit = make_pipeline(StandardScaler(), LogisticRegression(max_iter=2000)).fit(Xtr,ytr)
auc_logit = roc_auc_score(yte, logit.predict_proba(Xte)[:,1])
auc_tree  = roc_auc_score(yte, mstar.predict_proba(Xte)[:,1])
auc_rf    = roc_auc_score(yte, RandomForestClassifier(n_estimators=300,random_state=0,n_jobs=-1).fit(Xtr,ytr).predict_proba(Xte)[:,1])
print(f"OUT-OF-SAMPLE ROC-AUC:  logistic regression {auc_logit:.3f}   pruned CART {auc_tree:.3f}   random forest {auc_rf:.3f}")
# interpretable odds ratios via statsmodels Logit
sub=["PAY_1","LIMIT_BAL","AGE"]; Z=sm.add_constant(pd.DataFrame(Xtr,columns=feat)[sub])
lm=sm.Logit(ytr,Z).fit(disp=0); OR=np.exp(lm.params)
print("\nlogit odds ratios:  " + "   ".join(f"{k} {OR[k]:.3f}" for k in sub))
print(f"INTERPRETATION: each extra month in arrears (PAY_1) multiplies the odds of default by ~{OR['PAY_1']:.1f} -- a one-line")
print("statement a scorecard needs, which the tree cannot give directly.")
# probability of default vs months-in-arrears: empirical proportions vs PROBIT vs tree (functional form)
pay=Xtr[:,feat.index("PAY_1")]
vals=np.sort(np.unique(pay)); emp=np.array([ytr[pay==v].mean() for v in vals]); nobs=np.array([(pay==v).sum() for v in vals])
probit=sm.Probit(ytr, sm.add_constant(pay)).fit(disp=0)
grid=np.linspace(vals.min(),vals.max(),300); p_probit=probit.predict(sm.add_constant(grid))
p_tree=DecisionTreeClassifier(min_samples_leaf=300,random_state=0).fit(pay[:,None],ytr).predict_proba(grid[:,None])[:,1]
fig,ax=plt.subplots(figsize=(8.5,4.6))
ax.scatter(vals,emp,s=np.clip(nobs/25,15,400),color="k",zorder=3,label="empirical proportion (size ∝ n)")
ax.plot(grid,p_probit,color=ORANGE,lw=2,label="probit  P(default | PAY_1)")
ax.plot(grid,p_tree,color=BLUE,lw=2,label="tree  P(default | PAY_1)")
ax.axhline(y.mean(),color="grey",ls=":",lw=1); ax.set_xlabel("PAY_1  (months in arrears last month)"); ax.set_ylabel("probability of default")
ax.set_title("Default vs recent delinquency: empirical proportions, probit, and tree"); ax.legend(frameon=False)
plt.tight_layout(); plt.show()
print("Functional form on display: the empirical rate is FLAT for paid-up/revolving clients (PAY_1<=0) then JUMPS once in arrears --")
print("a kink, not a smooth trend. The PROBIT imposes a monotone S-curve through it; the univariate TREE steps to match the kink.")
print("Neither is 'right': probit buys a clean odds ratio, the tree buys flexibility. (Logit is nearly identical to probit here.)")
fig,ax=plt.subplots(figsize=(6.8,4))
ax.bar(["logistic\nregression","pruned\nCART","random\nforest"],[auc_logit,auc_tree,auc_rf],color=[ORANGE,BLUE,GREEN])
ax.set_ylim(0.5,0.82); ax.axhline(0.5,color="k",ls=":",lw=1); ax.set_ylabel("out-of-sample AUC")
ax.set_title("Traditional econometrics vs tree vs ensemble")
for i,v in enumerate([auc_logit,auc_tree,auc_rf]): ax.text(i,v+0.005,f"{v:.3f}",ha="center")
plt.tight_layout(); plt.show()
# predicted vs actual: calibration (reliability) + ROC
from sklearn.calibration import calibration_curve
from sklearn.metrics import roc_curve
p_lg=logit.predict_proba(Xte)[:,1]; p_tr=mstar.predict_proba(Xte)[:,1]
fig,ax=plt.subplots(1,2,figsize=(13,4.6))
for nm,pp,col,mk in [("logistic",p_lg,ORANGE,"o"),("pruned tree",p_tr,BLUE,"s")]:
    fr,mpv=calibration_curve(yte,pp,n_bins=10,strategy="quantile"); ax[0].plot(mpv,fr,mk+"-",color=col,label=nm)
ax[0].plot([0,.8],[0,.8],"k:",lw=1,label="perfect"); ax[0].set_xlabel("mean predicted P(default)"); ax[0].set_ylabel("observed default rate")
ax[0].set_title("Predicted vs actual (calibration by decile)"); ax[0].legend(frameon=False)
for nm,pp,col in [("logistic",p_lg,ORANGE),("pruned tree",p_tr,BLUE)]:
    fpr,tpr,_=roc_curve(yte,pp); ax[1].plot(fpr,tpr,color=col,label=f"{nm} (AUC {roc_auc_score(yte,pp):.3f})")
ax[1].plot([0,1],[0,1],"k:",lw=1); ax[1].set_xlabel("false-positive rate"); ax[1].set_ylabel("true-positive rate")
ax[1].set_title("ROC curves"); ax[1].legend(frameon=False,loc="lower right")
plt.tight_layout(); plt.show()
print("Predicted-vs-actual: both models track the diagonal (well-calibrated), but the tree's few leaves give only a HANDFUL")
print("of distinct probabilities (a steppy curve), while logit yields smooth, granular scores -- often preferred for pricing.")
print(f"\nThe tree edges plain logit ({auc_tree:.3f} vs {auc_logit:.3f}) by capturing nonlinearities and interactions the linear")
print("log-odds miss; the forest widens the gap. But logit is close, and far more interpretable -- and with domain-driven")
print("feature engineering (binning PAY_*, interactions) it closes much of the gap. Econometrics and ML are complements, not rivals.")

OUT-OF-SAMPLE ROC-AUC:  logistic regression 0.715   pruned CART 0.744   random forest 0.762

logit odds ratios:  PAY_1 2.002   LIMIT_BAL 1.000   AGE 1.008
INTERPRETATION: each extra month in arrears (PAY_1) multiplies the odds of default by ~2.0 -- a one-line
statement a scorecard needs, which the tree cannot give directly.

Functional form on display: the empirical rate is FLAT for paid-up/revolving clients (PAY_1<=0) then JUMPS once in arrears --
a kink, not a smooth trend. The PROBIT imposes a monotone S-curve through it; the univariate TREE steps to match the kink.
Neither is 'right': probit buys a clean odds ratio, the tree buys flexibility. (Logit is nearly identical to probit here.)

Predicted-vs-actual: both models track the diagonal (well-calibrated), but the tree's few leaves give only a HANDFUL
of distinct probabilities (a steppy curve), while logit yields smooth, granular scores -- often preferred for pricing.

The tree edges plain logit (0.744 vs 0.715) by capturing nonlinearities and interactions the linear
log-odds miss; the forest widens the gap. But logit is close, and far more interpretable -- and with domain-driven
feature engineering (binning PAY_*, interactions) it closes much of the gap. Econometrics and ML are complements, not rivals.

Decision Trees from Scratch — CART¶

Classification and Regression Trees (Breiman, Friedman, Olshen & Stone, 1984)¶

1. The data up close — what we are predicting¶

2. What a split is — impurity¶

3. Growing a tree, and reading it¶

4. From-scratch vs scikit-learn — do they agree?¶

5. Overfitting — the price of growing deep¶

6. Instability — why one tree is never enough¶

7. Cost-complexity pruning (Breiman's key idea)¶

8. Regression trees — the same engine, variance splits¶

9. What the tree learned — importances and decision surface¶

10. A traditional-econometrics benchmark — logistic regression¶

11. Summary¶