📊 Descriptive Statistics

Unit 7 — Extract real insights from data — grouping, pivoting, merging, time series, and statistical analysis.

The Full Statistical Picture

python
import pandas as pd, numpy as np np.random.seed(42) df = pd.read_csv("sales.csv") # or use sample data sales = pd.Series(np.random.normal(52000,15000,200).round(0)) # Complete summary print(sales.describe()) print(f"\nSkewness: {sales.skew():.3f}") print(f"Kurtosis: {sales.kurtosis():.3f}") print(f"CV: {sales.std()/sales.mean()*100:.1f}%")
Output
count 200.000 mean 52,156.000 std 15,284.000 min 9,781.000 25% 41,855.000 50% 52,244.000 75% 62,726.000 max 93,912.000 Skewness: 0.048 Kurtosis: -0.098 CV: 29.3%

Correlation Matrix

python
import pandas as pd, numpy as np np.random.seed(42) n = 100 df = pd.DataFrame({ "Sales": np.random.normal(50000,10000,n), "Marketing": np.random.normal(5000,1000,n), "Staff": np.random.randint(5,50,n), "Season": np.random.choice([1,2,3,4],n), }) # Add correlation df["Sales"] += df["Marketing"] * 4 corr = df.corr() print(corr.round(2)) # Find top correlated pairs import numpy as np mask = np.triu(np.ones(corr.shape), k=1).astype(bool) pairs = corr.where(mask).stack().sort_values(ascending=False) print("\nTop correlations:") print(pairs.head(3))
🎯 TASKPractice: Descriptive Statistics
Perform a complete descriptive analysis on a sales dataset: central tendency, spread, shape, and identify key insights.
Starter code:
python
import pandas as pd, numpy as np np.random.seed(42) df = pd.DataFrame({ "Region": np.random.choice(["Cairo","Alex","Giza","Aswan"],200), "Product": np.random.choice(["Laptop","Phone","Tablet"],200), "Sales": np.random.lognormal(10.8, 0.4, 200).round(0), "Units": np.random.randint(1,15,200), }) print("=== Overall Sales Statistics ===") print(f"Mean: {df['Sales'].mean():>12,.0f}") print(f"Median: {df['Sales'].median():>12,.0f}") print(f"Std Dev: {df['Sales'].std():>12,.0f}") print(f"Skew: {df['Sales'].skew():>12.3f}") print("\n=== By Region ===") by_region = df.groupby("Region")["Sales"].agg( Mean="mean", Median="median", Std="std", Count="count" ).round(0) print(by_region.sort_values("Mean", ascending=False)) print("\n=== Correlation ===") print(df[["Sales","Units"]].corr().round(3))

📊 GroupBy — Grouped Analysis

Unit 7 — Extract real insights from data — grouping, pivoting, merging, time series, and statistical analysis.

Basic GroupBy

python
import pandas as pd, numpy as np np.random.seed(42) df = pd.DataFrame({ "Region": np.random.choice(["Cairo","Alex","Giza"],100), "Product": np.random.choice(["Laptop","Phone","Tablet"],100), "Sales": np.random.randint(5000,80000,100), "Units": np.random.randint(1,10,100), }) # Group by one column print(df.groupby("Region")["Sales"].sum().sort_values(ascending=False)) # Multiple aggregations summary = df.groupby("Region")["Sales"].agg( Total="sum", Average="mean", Count="count", Max="max" ).round(0) print(summary)
Output
Region Cairo 1,245,000 Alex 987,000 Giza 823,000 Name: Sales, dtype: int64 Total Average Count Max Region Alex 987000 28229.0 35 79000 Cairo 1245000 35571.0 35 79000 Giza 823000 27433.0 30 77000

Named Aggregations & Transform

python
# Group by multiple columns pivot = df.groupby(["Region","Product"])["Sales"].agg(["sum","mean"]).round(0) print(pivot.head(6)) # transform — add group stats to each row df["Region_Avg"] = df.groupby("Region")["Sales"].transform("mean") df["vs_avg"] = df["Sales"] - df["Region_Avg"] df["Above_Avg"] = df["vs_avg"] > 0 # Filter groups (only regions with avg > 30000) high_regions = df.groupby("Region").filter(lambda x: x["Sales"].mean() > 30000) print(f"High regions: {high_regions['Region'].unique()}")
🎯 TASKPractice: GroupBy — Grouped Analysis
Analyze sales by region AND product. Find top product per region, regions missing from top 10 by revenue, and flag underperforming combinations.
Starter code:
python
import pandas as pd, numpy as np np.random.seed(42) df = pd.DataFrame({ "Region": np.random.choice(["Cairo","Alex","Giza","Aswan","Luxor"],200), "Product": np.random.choice(["Laptop","Phone","Tablet","Monitor"],200), "Sales": np.random.randint(3000,80000,200), "Quarter": np.random.choice(["Q1","Q2","Q3","Q4"],200), }) # Top product per region by_rp = df.groupby(["Region","Product"])["Sales"].sum() top_product = by_rp.groupby("Region").idxmax().apply(lambda x: x[1]) print("Top product per region:") print(top_product) # Overall totals per region region_totals = df.groupby("Region")["Sales"].sum().sort_values(ascending=False) print("\nRegion totals:") print(region_totals) # Underperforming: below 25th percentile for their region df["Region_Q25"] = df.groupby("Region")["Sales"].transform(lambda x: x.quantile(0.25)) weak = df[df["Sales"] < df["Region_Q25"]][["Region","Product","Sales"]].head(5) print("\nUnderperforming sales (sample):") print(weak.to_string(index=False))

📊 Merging DataFrames

Unit 7 — Extract real insights from data — grouping, pivoting, merging, time series, and statistical analysis.

Types of Joins

python
import pandas as pd orders = pd.DataFrame({ "Order_ID": [1,2,3,4,5], "Customer_ID":[101,102,101,103,999], "Sales": [5000,8000,3000,12000,7000], }) customers = pd.DataFrame({ "Customer_ID":[101,102,103,104], "Name": ["Ahmed","Sara","Omar","Layla"], "City": ["Cairo","Alex","Giza","Cairo"], }) # Left join — all orders, match customers left = pd.merge(orders, customers, on="Customer_ID", how="left") # Inner join — only matches inner = pd.merge(orders, customers, on="Customer_ID", how="inner") # Outer join — all from both outer = pd.merge(orders, customers, on="Customer_ID", how="outer") print(f"Orders: {len(orders)}") print(f"Left: {len(left)}") print(f"Inner: {len(inner)}") print(f"Outer: {len(outer)}") print(left)
Output
Orders: 5 Left: 5 Inner: 4 Outer: 6 Order_ID Customer_ID Sales Name City 0 1 101 5000 Ahmed Cairo 1 2 102 8000 Sara Alex 2 3 101 3000 Ahmed Cairo 3 4 103 12000 Omar Giza 4 5 999 7000 NaN NaN

Merging on Different Column Names

python
# Different column names in each DataFrame orders2 = pd.DataFrame({"OrdID":[1,2,3],"CustID":[101,102,103],"Sales":[5000,8000,3000]}) customers2 = pd.DataFrame({"ID":[101,102,103],"Name":["Ahmed","Sara","Omar"]}) merged = pd.merge(orders2, customers2, left_on = "CustID", right_on = "ID", how = "left" ) print(merged)
💡 Tip: Always check the row count after merging — an unexpected increase means duplicate keys in one of the DataFrames.
🎯 TASKPractice: Merging DataFrames
Merge 3 tables: orders, customers, and products. Calculate revenue per customer and identify top customers by city.
Starter code:
python
import pandas as pd orders = pd.DataFrame({ "Order_ID": range(1,11), "Cust_ID": [1,2,1,3,2,4,3,1,5,2], "Prod_ID": [101,102,103,101,102,103,101,102,103,101], "Quantity": [2,1,3,1,2,1,2,1,1,3], }) customers = pd.DataFrame({ "Cust_ID": [1,2,3,4,5], "Name": ["Ahmed","Sara","Omar","Layla","Karim"], "City": ["Cairo","Alex","Cairo","Giza","Alex"], }) products = pd.DataFrame({ "Prod_ID": [101,102,103], "Name": ["Laptop","Phone","Tablet"], "Price": [12000,6500,4200], }) # Merge all three merged = (orders .merge(customers, on="Cust_ID") .merge(products, on="Prod_ID", suffixes=("","_prod")) ) merged["Revenue"] = merged["Quantity"] * merged["Price"] # Top customers top_cust = merged.groupby(["Name","City"])["Revenue"].sum().sort_values(ascending=False) print("Revenue by customer:") print(top_cust) print("\nRevenue by city:") print(merged.groupby("City")["Revenue"].sum().sort_values(ascending=False))

📊 Concat & Join

Unit 7 — Extract real insights from data — grouping, pivoting, merging, time series, and statistical analysis.

pd.concat — Stack DataFrames

python
import pandas as pd # Vertical stack (add rows) q1 = pd.DataFrame({"Month":["Jan","Feb","Mar"],"Sales":[45000,52000,48000]}) q2 = pd.DataFrame({"Month":["Apr","May","Jun"],"Sales":[61000,58000,72000]}) all_data = pd.concat([q1, q2], ignore_index=True) print(all_data) # Horizontal stack (add columns) names = pd.DataFrame({"Name":["Ahmed","Sara","Omar"]}) scores = pd.DataFrame({"Score":[85,92,78],"Grade":["B","A","C"]}) combined = pd.concat([names, scores], axis=1) print(combined)
Output
Month Sales 0 Jan 45000 1 Feb 52000 2 Mar 48000 3 Apr 61000 4 May 58000 5 Jun 72000 Name Score Grade 0 Ahmed 85 B 1 Sara 92 A 2 Omar 78 C

Combining Multiple Files

python
import pandas as pd, glob, os # Load and combine multiple CSV files # files = glob.glob("sales_*.csv") # dfs = [pd.read_csv(f) for f in files] # all = pd.concat(dfs, ignore_index=True) # Add source column to track origin dfs = [q1.assign(quarter="Q1"), q2.assign(quarter="Q2")] all_q = pd.concat(dfs, ignore_index=True) print(all_q) # Stack different-shaped DataFrames (union of columns) df_a = pd.DataFrame({"A":[1,2],"B":[3,4]}) df_b = pd.DataFrame({"B":[5,6],"C":[7,8]}) stacked = pd.concat([df_a, df_b], ignore_index=True) print(stacked) # NaN where columns don't exist
💡 Tip: Add a source column before concat to track which file/period each row came from. Essential when combining multiple sheets or files.
🎯 TASKPractice: Concat & Join
Combine 4 quarterly DataFrames into one annual dataset. Add quarter labels, verify totals, and create a quarterly comparison.
Starter code:
python
import pandas as pd, numpy as np np.random.seed(42) quarters = [] for q in range(1,5): df = pd.DataFrame({ "Month": [f"{(q-1)*3+m:02d}" for m in range(1,4)], "Region": np.random.choice(["Cairo","Alex","Giza"],3), "Sales": np.random.randint(40000,80000,3), }) df["Quarter"] = f"Q{q}" quarters.append(df) # Combine all quarters annual = pd.concat(quarters, ignore_index=True) print(f"Total rows: {len(annual)}") print(f"Quarters: {annual['Quarter'].unique()}") # Quarterly comparison q_summary = annual.groupby("Quarter")["Sales"].agg(["sum","mean"]).round(0) print("\nQuarterly Summary:") print(q_summary) # Verify: sum of quarters = annual total print(f"\nAnnual total: {annual['Sales'].sum():,}") print(f"Q-totals sum: {q_summary['sum'].sum():,}")

📊 Pivot Tables

Unit 7 — Extract real insights from data — grouping, pivoting, merging, time series, and statistical analysis.

Creating Pivot Tables

python
import pandas as pd, numpy as np np.random.seed(42) df = pd.DataFrame({ "Region": np.random.choice(["Cairo","Alex","Giza"],100), "Quarter": np.random.choice(["Q1","Q2","Q3","Q4"],100), "Product": np.random.choice(["Laptop","Phone","Tablet"],100), "Sales": np.random.randint(5000,80000,100), }) # Basic pivot table pivot = df.pivot_table( values = "Sales", index = "Region", columns = "Quarter", aggfunc = "sum", margins = True, # Add totals fill_value = 0, ) print(pivot.round(0))
Output
Quarter Q1 Q2 Q3 Q4 All Region Alex 234000 198000 256000 287000 975000 Cairo 312000 345000 298000 321000 1276000 Giza 198000 212000 178000 245000 833000 All 744000 755000 732000 853000 3084000

Multiple Values & Functions

python
# Multiple aggfunc multi = df.pivot_table( values = "Sales", index = "Region", columns = "Product", aggfunc = ["sum","mean","count"], ).round(0) # Cross-tabulation (counts) ct = pd.crosstab(df["Region"], df["Quarter"], values=df["Sales"], aggfunc="sum", normalize="index" # Row percentages ) * 100 print(ct.round(1))
💡 Tip: Use margins=True to automatically add totals row and column. Use fill_value=0 to replace NaN with 0 in the pivot table.
🎯 TASKPractice: Pivot Tables
Create a comprehensive pivot analysis: revenue by region/quarter, market share by product, and identify best region-product combinations.
Starter code:
python
import pandas as pd, numpy as np np.random.seed(42) df = pd.DataFrame({ "Region": np.random.choice(["Cairo","Alex","Giza","Aswan"],200), "Quarter": np.random.choice(["Q1","Q2","Q3","Q4"],200), "Product": np.random.choice(["Laptop","Phone","Tablet","Monitor"],200), "Sales": np.random.randint(3000,80000,200), "Units": np.random.randint(1,10,200), }) # Revenue pivot rev_pivot = df.pivot_table( values="Sales", index="Region", columns="Quarter", aggfunc="sum", margins=True, fill_value=0 ) print("Revenue by Region/Quarter:") print(rev_pivot.round(0)) # Market share by product product_share = (df.groupby("Product")["Sales"].sum() / df["Sales"].sum() * 100).round(1) print("\nProduct market share %:") print(product_share.sort_values(ascending=False)) # Best combo combo = df.groupby(["Region","Product"])["Sales"].sum() print(f"\nBest combo: {combo.idxmax()} ({combo.max():,})")

📊 Time Series Basics

Unit 7 — Extract real insights from data — grouping, pivoting, merging, time series, and statistical analysis.

Date Index & Resampling

python
import pandas as pd, numpy as np np.random.seed(42) # Create daily time series dates = pd.date_range("2024-01-01","2024-12-31",freq="D") df = pd.DataFrame({ "Sales": np.random.normal(50000,10000,len(dates)).round(0), "Units": np.random.randint(1,20,len(dates)), }, index=dates) # Extract date components df["Month"] = df.index.month df["Quarter"] = df.index.quarter df["DayName"] = df.index.day_name() # Resample — aggregate by different frequencies monthly = df["Sales"].resample("ME").sum() weekly = df["Sales"].resample("W").mean() quarterly = df["Sales"].resample("QE").agg(["sum","mean","count"]) print("Monthly Sales (first 3):") print(monthly.head(3).round(0)) print("\nQuarterly:") print(quarterly.round(0))
Output
Monthly Sales (first 3): 2024-01-31 1478234 2024-02-29 1389456 2024-03-31 1523891 Quarterly: sum mean count 2024-03-31 4391581 48260.0 91 2024-06-30 4512345 49585.0 91
🎯 TASKPractice: Time Series Basics
Analyze a year of daily sales: find seasonality patterns, best/worst day of week, month-over-month growth, and detect trend.
Starter code:
python
import pandas as pd, numpy as np np.random.seed(42) dates = pd.date_range("2023-01-01","2024-12-31",freq="D") # Add trend + seasonality + noise trend = np.linspace(40000,60000,len(dates)) seasonal = np.sin(np.linspace(0,4*np.pi,len(dates))) * 5000 noise = np.random.normal(0,3000,len(dates)) df = pd.DataFrame({"Sales": (trend+seasonal+noise).round(0)}, index=dates) # Monthly totals and growth monthly = df["Sales"].resample("ME").sum() growth = monthly.pct_change() * 100 # Day of week analysis df["DayName"] = df.index.day_name() by_day = df.groupby("DayName")["Sales"].mean().sort_values(ascending=False) # Best and worst months print(f"Best month: {monthly.idxmax().strftime('%Y-%m')} ({monthly.max():,.0f})") print(f"Worst month: {monthly.idxmin().strftime('%Y-%m')} ({monthly.min():,.0f})") print(f"\nAvg growth: {growth.mean():.2f}% per month") print(f"\nBest day of week: {by_day.index[0]} ({by_day.iloc[0]:,.0f})") print(f"Worst day: {by_day.index[-1]} ({by_day.iloc[-1]:,.0f})")

📊 Window Functions

Unit 7 — Extract real insights from data — grouping, pivoting, merging, time series, and statistical analysis.

Rolling Windows

python
import pandas as pd, numpy as np monthly = pd.Series([45,52,48,61,58,72,65,59,71,78,82,95], index=pd.date_range("2024-01",periods=12,freq="ME"), name="Sales" ) * 1000 # Rolling mean (smoothing) ma3 = monthly.rolling(window=3).mean() # 3-month moving avg ma6 = monthly.rolling(window=6).mean() # 6-month moving avg # Rolling statistics roll_std = monthly.rolling(window=3).std() roll_min = monthly.rolling(window=3).min() roll_max = monthly.rolling(window=3).max() result = pd.DataFrame({ "Sales": monthly, "MA3": ma3, "MA6": ma6, "Std3": roll_std, }).round(0) print(result.head(8))
Output
Sales MA3 MA6 Std3 2024-01-31 45000 NaN NaN NaN 2024-02-29 52000 NaN NaN NaN 2024-03-31 48000 48333 NaN 3512 2024-04-30 61000 53667 NaN 6807 2024-05-31 58000 55667 NaN 6807 2024-06-30 72000 63667 56000 7211

Expanding & Cumulative

python
# Expanding — includes all previous rows cum_mean = monthly.expanding().mean() cum_total = monthly.cumsum() cum_max = monthly.cummax() # Percentage change pct = monthly.pct_change() * 100 result2 = pd.DataFrame({ "Sales": monthly, "CumTotal": cum_total, "RunningAvg":cum_mean, "Growth%": pct, }).round(0) print(result2.tail(5))
💡 Tip: Rolling averages smooth out noise in time series. Use 3-period MA for short-term trend, 12-period for long-term.
🎯 TASKPractice: Window Functions
Apply window functions to daily sales: calculate 7-day and 30-day moving averages, detect days where sales spike > 2 std above rolling mean, and calculate cumulative revenue milestones.
Starter code:
python
import pandas as pd, numpy as np np.random.seed(42) dates = pd.date_range("2024-01-01","2024-06-30",freq="D") sales = pd.Series( np.random.normal(50000,8000,len(dates)).round(0), index=dates, name="Sales" ) # Moving averages ma7 = sales.rolling(7).mean() ma30 = sales.rolling(30).mean() # Rolling std for spike detection roll_std = sales.rolling(30).std() roll_mean = sales.rolling(30).mean() spikes = sales[sales > roll_mean + 2*roll_std] # Cumulative revenue cum_rev = sales.cumsum() milestones = [1e6, 2e6, 3e6] for m in milestones: date = cum_rev[cum_rev >= m].index[0] if (cum_rev >= m).any() else None if date: print(f"Reached {m/1e6:.0f}M on: {date.date()}") print(f"\nSpike days: {len(spikes)}") print(f"Avg spike amount: {spikes.mean():,.0f}") print(f"\nFinal 7-day MA: {ma7.iloc[-1]:,.0f}")

📊 Correlation & Regression

Unit 7 — Extract real insights from data — grouping, pivoting, merging, time series, and statistical analysis.

Correlation Analysis

python
import pandas as pd, numpy as np np.random.seed(42) n = 100 marketing = np.random.normal(5000, 1000, n) sales = marketing * 8 + np.random.normal(0, 5000, n) # Correlated df = pd.DataFrame({ "Marketing": marketing.round(0), "Sales": sales.round(0), "Staff": np.random.randint(5, 30, n), "Season": np.random.choice([1,2,3,4], n), }) # Pearson correlation print("Correlation matrix:") print(df.corr().round(3)) # Spearman (rank-based, robust to outliers) print("\nSpearman correlation:") print(df.corr(method="spearman").round(3))
Output
Correlation matrix: Marketing Sales Staff Season Marketing 1.000 0.845 0.012 -0.023 Sales 0.845 1.000 0.089 -0.041 Staff 0.012 0.089 1.000 0.054 Season -0.023 -0.041 0.054 1.000

Simple Linear Regression

python
import numpy as np # Manual linear regression X = df["Marketing"].values y = df["Sales"].values # OLS: y = a + bX b = np.cov(X, y)[0,1] / np.var(X) a = y.mean() - b * X.mean() # Predictions y_pred = a + b * X residuals = y - y_pred r_squared = 1 - (residuals**2).sum() / ((y - y.mean())**2).sum() print(f"Intercept (a): {a:,.0f}") print(f"Slope (b): {b:.2f}") print(f"R²: {r_squared:.3f}") print(f"\nFor 6000 marketing: {a + b*6000:,.0f} predicted sales")
🎯 TASKPractice: Correlation & Regression
Perform correlation analysis on a multi-variable dataset. Find the variables most correlated with sales, build a simple regression, and interpret results.
Starter code:
python
import pandas as pd, numpy as np np.random.seed(42) n = 150 # Generate correlated data marketing = np.random.normal(5000,1000,n) staff = np.random.randint(5,30,n) season_eff = np.random.choice([0.8,1.0,1.2,1.1],n) sales = (marketing*7 + staff*500 + np.random.normal(0,3000,n)) * season_eff df = pd.DataFrame({ "Sales": sales.round(0), "Marketing": marketing.round(0), "Staff": staff, "SeasonEff": season_eff, "Temperature": np.random.normal(25,5,n).round(1), }) # Correlation with Sales corr_with_sales = df.corr()["Sales"].drop("Sales").sort_values(ascending=False) print("Correlation with Sales:") print(corr_with_sales.round(3)) # Simple regression: Sales ~ Marketing X = df["Marketing"].values y = df["Sales"].values b = np.cov(X,y)[0,1]/np.var(X) a = y.mean()-b*X.mean() y_pred = a+b*X r2 = 1-((y-y_pred)**2).sum()/((y-y.mean())**2).sum() print(f"\nRegression: Sales = {a:,.0f} + {b:.2f} × Marketing") print(f"R² = {r2:.3f} ({r2*100:.1f}% variance explained)")

📊 Sampling & Hypothesis Testing

Unit 7 — Extract real insights from data — grouping, pivoting, merging, time series, and statistical analysis.

Random Sampling

python
import pandas as pd, numpy as np np.random.seed(42) # Large population population = pd.DataFrame({ "Sales": np.random.normal(50000,15000,10000).round(0), "Region": np.random.choice(["Cairo","Alex","Giza"],10000), }) # Simple random sample sample = population.sample(n=100, random_state=42) print(f"Population mean: {population['Sales'].mean():,.0f}") print(f"Sample mean: {sample['Sales'].mean():,.0f}") # Stratified sample (proportional by region) stratified = population.groupby("Region", group_keys=False).apply( lambda x: x.sample(frac=0.01, random_state=42) ) print(f"\nStratified sample: {len(stratified)} rows") print(stratified["Region"].value_counts())
Output
Population mean: 50,012 Sample mean: 49,847 Stratified sample: 100 rows Region Cairo 34 Alex 33 Giza 33

Hypothesis Testing (t-test)

python
import numpy as np from scipy import stats # Are Cairo sales significantly higher than Alex? cairo = population[population["Region"]=="Cairo"]["Sales"] alex = population[population["Region"]=="Alex"]["Sales"] t_stat, p_value = stats.ttest_ind(cairo, alex) print(f"Cairo mean: {cairo.mean():,.0f}") print(f"Alex mean: {alex.mean():,.0f}") print(f"t-statistic: {t_stat:.3f}") print(f"p-value: {p_value:.4f}") print(f"Significant (p<0.05): {p_value < 0.05}")
💡 Tip: p-value < 0.05 means we reject the null hypothesis — the difference is statistically significant (not due to random chance).
🎯 TASKPractice: Sampling & Hypothesis Testing
Compare sales between two marketing campaigns using hypothesis testing. Determine if the difference is statistically significant.
Starter code:
python
import numpy as np, pandas as pd from scipy import stats np.random.seed(42) # Two campaigns campaign_A = np.random.normal(52000, 8000, 50) # New campaign campaign_B = np.random.normal(48000, 9000, 50) # Old campaign # Descriptive stats print("Campaign A:") print(f" Mean: {campaign_A.mean():,.0f} Std: {campaign_A.std():,.0f}") print("Campaign B:") print(f" Mean: {campaign_B.mean():,.0f} Std: {campaign_B.std():,.0f}") print(f" Difference: {campaign_A.mean()-campaign_B.mean():,.0f}") # Independent t-test t_stat, p_val = stats.ttest_ind(campaign_A, campaign_B) print(f"\nt-statistic: {t_stat:.3f}") print(f"p-value: {p_val:.4f}") if p_val < 0.05: print("✅ SIGNIFICANT: Campaign A performs better (p < 0.05)") else: print("❌ NOT significant: Could be random variation (p >= 0.05)") # Effect size (Cohen's d) pooled_std = np.sqrt((campaign_A.std()**2 + campaign_B.std()**2) / 2) cohens_d = (campaign_A.mean() - campaign_B.mean()) / pooled_std print(f"Cohen's d: {cohens_d:.3f} ({'large' if abs(cohens_d)>0.8 else 'medium' if abs(cohens_d)>0.5 else 'small'} effect)")

📊 Building Analysis Reports

Unit 7 — Extract real insights from data — grouping, pivoting, merging, time series, and statistical analysis.

Structuring an Analysis Report

python
import pandas as pd, numpy as np np.random.seed(42) def sales_report(df, title="Sales Report"): """Generate a complete sales analysis report.""" sep = "="*50 print(f"\n{sep}") print(f" {title}") print(sep) # 1. Overview print(f"\n📊 OVERVIEW") print(f" Period: {df.index.min().date()} to {df.index.max().date()}") print(f" Total: {df['Sales'].sum():>15,.0f}") print(f" Daily Avg: {df['Sales'].mean():>15,.0f}") print(f" Best Day: {df['Sales'].max():>15,.0f} ({df['Sales'].idxmax().date()})") # 2. Trend monthly = df["Sales"].resample("ME").sum() growth = monthly.pct_change().mean() * 100 print(f"\n📈 TREND") print(f" Monthly growth (avg): {growth:+.1f}%") # 3. Regional breakdown if "Region" in df.columns: print(f"\n🗺️ BY REGION") by_region = df.groupby("Region")["Sales"].agg(["sum","mean"]).round(0) by_region["Share%"] = (by_region["sum"]/by_region["sum"].sum()*100).round(1) print(by_region.sort_values("sum",ascending=False).to_string()) print(f"\n{sep}\n") return monthly
💡 Tip: A good report has: Overview → Trend → Breakdown → Insights → Recommendations. Always answer the 'so what' question.
🎯 TASKPractice: Building Analysis Reports
Build a full quarterly business report with: executive summary, KPI dashboard, regional analysis, product performance, and growth analysis.
Starter code:
python
import pandas as pd, numpy as np np.random.seed(42) # Generate dataset dates = pd.date_range("2024-01-01","2024-12-31",freq="D") regions = np.random.choice(["Cairo","Alex","Giza","Aswan"],len(dates)) products = np.random.choice(["Laptop","Phone","Tablet"],len(dates)) sales = np.random.normal(50000,15000,len(dates)).round(0) df = pd.DataFrame({"Sales":sales,"Region":regions,"Product":products},index=dates) print("=" * 55) print(" ANNUAL BUSINESS REPORT — 2024") print("=" * 55) # KPIs print(f"\n{'KPI':<25} {'Value':>15}") print("-"*40) print(f"{'Total Revenue':<25} {df['Sales'].sum():>15,.0f}") print(f"{'Daily Average':<25} {df['Sales'].mean():>15,.0f}") print(f"{'Best Month Revenue':<25}", end="") monthly=df["Sales"].resample("ME").sum() print(f"{monthly.max():>15,.0f}") print(f"{'YoY Growth Est.':<25} {'N/A - 1 year':>15}") # Regional breakdown print(f"\n{'REGIONAL PERFORMANCE':=^40}") reg=df.groupby("Region")["Sales"].agg(Total="sum",Daily="mean").round(0) reg["Share%"]=(reg["Total"]/reg["Total"].sum()*100).round(1) print(reg.sort_values("Total",ascending=False).to_string()) # Product mix print(f"\n{'PRODUCT MIX':=^40}") prod=df.groupby("Product")["Sales"].sum() for p,v in prod.sort_values(ascending=False).items(): share=v/prod.sum()*100 bar="█"*int(share/5) print(f" {p:<10} {v:>10,.0f} ({share:.1f}%) {bar}")