Python Data Manipulation with Pandas Questions

import pandas as pd

def _normalize_columns(df):
    return df.rename(columns=lambda c: c.strip().lower().replace(' ', '_'))

def _enforce_dtypes(df, schema):
    # schema: dict col -> dtype (e.g. 'float', 'int', 'category', 'datetime')
    for col, dtype in schema.items():
        if col in df:
            if dtype == 'datetime':
                df[col] = pd.to_datetime(df[col], errors='coerce')
            else:
                df[col] = df[col].astype(dtype, errors='ignore')
    return df

def _apply_fill_policy(df, fill_policy):
    # fill_policy: dict col -> value or callable(column_series)->value
    for col, policy in fill_policy.items():
        if col not in df:
            continue
        val = policy(df[col]) if callable(policy) else policy
        df[col] = df[col].fillna(val)
    return df

def normalize_df(df, schema, fill_policy, inplace=False):
    df = df if inplace else df.copy()
    df = _normalize_columns(df)
    df = _enforce_dtypes(df, schema)
    df = _apply_fill_policy(df, fill_policy)
    return df

import pandera as pa
from pandera import Column, DataFrameSchema, Check
import pandas as pd

schema = DataFrameSchema({
    "id": Column(int, nullable=False, unique=True),
    "timestamp": Column(pa.DateTime, nullable=False),
    "age": Column(int, nullable=False, checks=[
        Check.ge(0), Check.le(120)]),
    "income": Column(float, nullable=True, checks=Check.ge(0)),
    "country": Column(str, nullable=False, checks=Check.isin(["US","CA","UK"]))
})

# custom complex check: ratio of nulls under threshold
def null_ratio_ok(df: pd.DataFrame) -> bool:
    return (df.isnull().mean() < 0.05).all()

schema.add_check(Check(null_ratio_ok, element_wise=False))
validated = schema.validate(df, lazy=True)  # collects all errors

import pandas as pd

# Example input: mixed strings
data = [
  "2023-03-26 01:30:00",            # naive (Europe/Berlin DST spring transition)
  "2023-10-29 02:30:00+02:00",      # aware with offset
  "2023-11-05 01:30:00"             # naive (possible ambiguous in other zones)
]
s = pd.Series(data)

# 1) Parse strings; aware offsets are preserved, naive become naive Timestamp objects
parsed = pd.to_datetime(s)

# Separate naive vs aware
naive_mask = parsed.dt.tz is None  # can't use this directly on Series; use .dt.tzinfo check per element
# Simpler: try localizing all naive values by creating a naive-only Series
naive_idx = parsed.map(lambda x: x.tzinfo is None)
naive = parsed[naive_idx]

# Localize with explicit DST handling:
# ambiguous: can be 'infer', bool array, 'NaT', or 'raise' (default)
# nonexistent: 'shift_forward', 'shift_backward', 'NaT', or 'raise'
localized = naive.dt.tz_localize(
    "Europe/Berlin",
    ambiguous="infer",        # try to infer from order; alternately True/False or array
    nonexistent="shift_forward"  # move clocks forward into the next valid time
)

# Put back together: convert already-aware entries to Zone-aware Europe/Berlin if they carry offsets
aware = parsed[~naive_idx].dt.tz_convert("UTC").dt.tz_localize(None)  # example handling if needed

# Combine localized and already-aware (here: ensure everything is timezone-aware)
combined = pd.concat([
    localized, 
    parsed[~naive_idx].dt.tz_convert("Europe/Berlin")  # convert offset-aware to the zone
]).sort_index()

# Finally normalize to UTC for aggregation
utc = combined.dt.tz_convert("UTC")

import dask.dataframe as dd
# read
ddf = dd.read_parquet("s3://bucket/data/*.parquet")
# previously: df = pd.read_parquet(...)
# then keep same pandas-like chain, compute at end
result = ddf.groupby("key")["val"].sum().compute()

import modin.pandas as pd  # minimal drop-in for many pandas calls
df = pd.read_parquet("data.parquet")
res = df.groupby("key")["val"].sum()

import pandas as pd
df = pd.read_parquet('data.parquet')  # or however loaded
print(df['country'].memory_usage(deep=True))

df['country'] = df['country'].astype('category')
print(df['country'].cat.categories.size)   # should be ~200
print(df['country'].memory_usage(deep=True))