# -*- coding: utf-8 -*-
"""Fetch a 'factor zoo' from the Ken French data library and assemble a monthly
panel of long-short factor returns -> factor_zoo.csv in the ex3 folder."""
import subprocess, zipfile, io, os, re, numpy as np, pandas as pd
BASE="https://mba.tuck.dartmouth.edu/pages/faculty/ken.french/ftp/"
# both paths resolve against this file, so the script runs wherever the folder is checked out
HERE=os.path.dirname(os.path.abspath(__file__))
TMP=os.path.join(HERE, "_kf")           # scratch space for the downloaded zips
OUT=os.path.join(HERE, "factor_zoo.csv")
os.makedirs(TMP, exist_ok=True)

def dl(zipname):
    path=os.path.join(TMP, zipname)
    subprocess.run(["curl","-sL","-o",path,BASE+zipname], check=True)
    with zipfile.ZipFile(path) as z:
        nm=[n for n in z.namelist() if n.lower().endswith(".csv")][0]
        return z.read(nm).decode("latin-1")

def parse_monthly(text):
    """First monthly block (value-weighted) as a DataFrame indexed by YYYYMM int."""
    lines=text.splitlines()
    hdr=None
    for i,l in enumerate(lines):
        if l.strip().startswith(",") and any(ch.isalpha() for ch in l):
            hdr=i; break
    cols=[c.strip() for c in lines[hdr].split(",")]
    rows=[]
    for l in lines[hdr+1:]:
        m=re.match(r"^\s*(\d{6})\s*,",l)
        if not m: break                                  # stop at first non-monthly (blank / annual / next block)
        vals=[x.strip() for x in l.split(",")]
        rows.append([int(vals[0])]+[float(x) for x in vals[1:len(cols)]])
    df=pd.DataFrame(rows, columns=["ym"]+cols[1:]).set_index("ym")
    return df.replace([-99.99,-999,-99.99],np.nan)

# --- factor files (single-column factors, already long-short) ---
FACTORS={}
ff5=parse_monthly(dl("F-F_Research_Data_5_Factors_2x3_CSV.zip"))
for c in ["Mkt-RF","SMB","HML","RMW","CMA"]: FACTORS[c]=ff5[c]
RF=ff5["RF"]
FACTORS["Mom"]=parse_monthly(dl("F-F_Momentum_Factor_CSV.zip")).iloc[:,0]
FACTORS["ST_Rev"]=parse_monthly(dl("F-F_ST_Reversal_Factor_CSV.zip")).iloc[:,0]
FACTORS["LT_Rev"]=parse_monthly(dl("F-F_LT_Reversal_Factor_CSV.zip")).iloc[:,0]

# --- decile portfolio files -> long-short (Hi 10 - Lo 10), a deliberately redundant zoo ---
DECILE={"Value_BEME":"Portfolios_Formed_on_BE-ME_CSV.zip",
        "Size_ME":"Portfolios_Formed_on_ME_CSV.zip",
        "Prof_OP":"Portfolios_Formed_on_OP_CSV.zip",
        "Inv_INV":"Portfolios_Formed_on_INV_CSV.zip",
        "Earn_EP":"Portfolios_Formed_on_E-P_CSV.zip",
        "Cash_CFP":"Portfolios_Formed_on_CF-P_CSV.zip",
        "Div_DP":"Portfolios_Formed_on_D-P_CSV.zip",
        "Accr_AC":"Portfolios_Formed_on_AC_CSV.zip",
        "NetIss_NI":"Portfolios_Formed_on_NI_CSV.zip",
        "Var":"Portfolios_Formed_on_VAR_CSV.zip",
        "ResVar":"Portfolios_Formed_on_RESVAR_CSV.zip",
        "Beta":"Portfolios_Formed_on_BETA_CSV.zip"}
for name,zp in DECILE.items():
    try:
        df=parse_monthly(dl(zp)); cols={c.strip():c for c in df.columns}
        hi=df[cols.get("Hi 10","Hi 10")]; lo=df[cols.get("Lo 10","Lo 10")]
        FACTORS[name]=hi-lo
    except Exception as e:
        print("SKIP",name,repr(e)[:80])

panel=pd.DataFrame(FACTORS)
panel["RF"]=RF
panel=panel.dropna()
panel=panel[(panel.index>=196307)]
panel.to_csv(OUT)
print("saved", OUT, panel.shape)
print("date range:", panel.index.min(), "-", panel.index.max())
print("factors:", [c for c in panel.columns if c!="RF"])
print(panel.drop(columns="RF").mean().round(3).to_string())      # monthly mean returns (% ), sanity