import pandas as pd
import numpy as np
from scipy import stats

df = pd.read_csv("house_prices.csv")

# ── Phase 1: First Pass ────────────────────────────────
print(df.shape)               # (1460, 81) → rows, columns
print(df.dtypes.value_counts()) # how many numeric vs object cols
print(df.isnull().sum().sort_values(ascending=False)[:10])
print(df.nunique().sort_values())   # columns with few unique = likely categorical

# ── Phase 2: Summary Statistics ───────────────────────
df.describe()         # count, mean, std, min, 25%, 50%, 75%, max
df.describe(include="all")  # also shows top/freq for object cols

# ── Phase 3: Individual column statistics ─────────────
col = df["SalePrice"]

mean   = col.mean()       # arithmetic mean — sensitive to outliers
median = col.median()     # middle value — robust to outliers
mode   = col.mode()[0]   # most frequent value
std    = col.std()        # standard deviation
var    = col.var()        # variance = std²
q1     = col.quantile(0.25)
q3     = col.quantile(0.75)
iqr    = q3 - q1         # interquartile range
cv     = std / mean       # coefficient of variation (relative spread)

print(f"Mean: {mean:,.0f}  Median: {median:,.0f}  Diff: {mean-median:,.0f}")
print(f"IQR: {iqr:,.0f}  CV: {cv:.3f}")

# ── Phase 4: Shape statistics ─────────────────────────
skewness = col.skew()           # 0=symmetric, >0=right tail, 
kurtosis = col.kurtosis()       # 0=normal tails, >0=heavy tails
print(f"Skewness: {skewness:.3f}  Kurtosis: {kurtosis:.3f}")

# ── Phase 5: Value counts for categoricals ────────────
print(df["Neighborhood"].value_counts())
print(df["Neighborhood"].value_counts(normalize=True).round(3))

# ── Automation: stats for ALL numeric columns ─────────
numeric_cols = df.select_dtypes("number").columns
stats_df = df[numeric_cols].agg(["mean", "median", "std",
    lambda x: x.skew(), lambda x: x.isnull().mean()]).T
stats_df.columns = ["mean", "median", "std", "skewness", "null_pct"]
print(stats_df.sort_values("skewness", ascending=False).head(10))

import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats
import numpy as np

# ── Visualise distribution ────────────────────────────
fig, axes = plt.subplots(1, 3, figsize=(15, 4))

# Histogram + KDE
sns.histplot(df["SalePrice"], bins=40, kde=True, ax=axes[0])
axes[0].set_title(f"SalePrice (skew={df['SalePrice'].skew():.2f})")

# QQ-plot: if points lie on diagonal line → normal distribution
stats.probplot(df["SalePrice"].dropna(), plot=axes[1])
axes[1].set_title("QQ-Plot: Deviation from Normal")

# Log transform: right-skewed → more symmetric
log_price = np.log1p(df["SalePrice"])
sns.histplot(log_price, bins=40, kde=True, ax=axes[2])
axes[2].set_title(f"log(SalePrice) (skew={log_price.skew():.2f})")
plt.tight_layout()

# ── Formal normality test ─────────────────────────────
# H0: data is normally distributed. p > 0.05 = fail to reject
stat, p = stats.shapiro(df["SalePrice"].dropna().sample(500, random_state=42))
print(f"Shapiro-Wilk: stat={stat:.4f}, p={p:.6f}")
# Very small p → strongly non-normal (expected for house prices)

# ── Box-Cox optimal transformation ───────────────────
# lambda ~0 = log, ~0.5 = sqrt, ~1 = no transform
pos = df["SalePrice"].dropna()
transformed, lam = stats.boxcox(pos)
print(f"Optimal lambda: {lam:.3f} → "
      f"{'log' if abs(lam) )

# ── Find all highly skewed columns ────────────────────
skewed = df.select_dtypes("number").skew().sort_values(ascending=False)
high_skew = skewed[abs(skewed) > 1]
print(f"{len(high_skew)} columns with |skewness| > 1")
print(high_skew.head(10))

Distribution Types and Actions