Python(Pandas分类变量内存优化):海量数据表category类型降内存实战

import pandas as pdimport numpy as np# 构造百万行测试数据n = 1000000data = {    "log_id": np.arange(n),    "area": np.random.choice(["北京","上海","广州","成都"], size=n),    "biz_type": np.random.choice(["支付","登录","查询"], size=n),    "status": np.random.choice(["成功","失败"], size=n),    "create_time": pd.date_range("2026-01-01", periods=n, freq="1s")}df = pd.DataFrame(data)# 查看原始内存print("转换前内存：")print(df[["area","biz_type","status"]].memory_usage(deep=True))# 将高重复字符串转为category类型df["area"] = df["area"].astype("category")df["biz_type"] = df["biz_type"].astype("category")df["status"] = df["status"].astype("category")print("\n转换后内存：")print(df[["area","biz_type","status"]].memory_usage(deep=True))# category正常筛选、分组示例df_success = df[df["status"] == "成功"]group_result = df.groupby(["area","biz_type"]).size()print("\n分组统计结果：")print(group_result.head())

转换前内存：Index            132area        14000000biz_type    14000000status      14000000dtype: int64转换后内存：Index           132area        1000057biz_type    1000043status      1000029dtype: int64分组统计结果：area  biz_type上海    支付          83537      查询          83348      登录          83626北京    支付          83284      查询          83521dtype: int64