# Ubuntu/Debian
sudo apt update && sudo apt install python3.12 python3.12-venv python3.12-dev

# macOS（使用Homebrew）
brew install python@3.12

# Windows
# 从官网下载：https://www.python.org/downloads/release/python-3122/

curl -sSL https://install.python-poetry.org | python3 -

# 验证安装
poetry --version  # 应输出1.8.x或更高版本

# 创建项目目录
mkdir data-analysis-2026 && cd data-analysis-2026

# 初始化项目
poetry init -n

# 安装核心依赖
poetry add pandas numpy polars matplotlib seaborn plotly scikit-learn jupyterlab openpyxl

# 启动Jupyter Lab
poetry run jupyter lab

import pandas as pd
import polars as pl
import numpy as np
from datetime import datetime

# 使用Polars高效加载大型数据集（2026年推荐）
df = pl.read_csv("ecommerce_behavior_202601.csv")

# 转换为Pandas DataFrame（如果需要兼容旧代码）
df_pd = df.to_pandas()

# 查看数据基本信息
print(f"数据集大小: {df.shape}")
print(df.head())

# 检查缺失值
print(df.null_count())

# 处理策略：
# 1. 数值型字段缺失：用中位数填充
# 2. 分类型字段缺失：用众数填充
# 3. 关键字段缺失（如user_id、behavior_type）：直接删除

df_clean = df.with_columns(
    # 填充年龄缺失值
    pl.col("age").fill_null(pl.col("age").median()),
    # 填充地区缺失值
    pl.col("region").fill_null(pl.col("region").mode()[0])
).drop_nulls(subset=["user_id", "item_id", "behavior_type"])

print(f"清洗后数据量: {df_clean.shape}")

# 检查重复值
duplicate_count = df_clean.is_duplicated().sum()
print(f"重复数据量: {duplicate_count}")

# 删除重复值
df_clean = df_clean.unique()

# 转换时间戳为日期时间格式
df_clean = df_clean.with_columns(
    pl.from_epoch("timestamp", time_unit="s").alias("event_time")
)

# 提取日期、小时、星期信息
df_clean = df_clean.with_columns(
    pl.col("event_time").dt.date().alias("date"),
    pl.col("event_time").dt.hour().alias("hour"),
    pl.col("event_time").dt.weekday().alias("weekday")
)

# 处理价格异常值（0或负数）
df_clean = df_clean.filter(pl.col("price") > 0)

# 使用IQR方法处理价格极端值
q1 = df_clean.select(pl.col("price").quantile(0.25)).item()
q3 = df_clean.select(pl.col("price").quantile(0.75)).item()
iqr = q3 - q1
lower_bound = q1 - 1.5 * iqr
upper_bound = q3 + 1.5 * iqr

df_clean = df_clean.filter(
    (pl.col("price") >= lower_bound) & (pl.col("price") <= upper_bound)
)

# 统计各行为类型数量
behavior_stats = df_clean.group_by("behavior_type").agg(
    pl.count("user_id").alias("count")
).sort("count", descending=True)

print(behavior_stats)

# 计算转化率
total_pv = behavior_stats.filter(pl.col("behavior_type") == "pv").select("count").item()
total_buy = behavior_stats.filter(pl.col("behavior_type") == "buy").select("count").item()
conversion_rate = total_buy / total_pv * 100

print(f"整体转化率: {conversion_rate:.2f}%")

# 按小时分析用户活跃度
hourly_activity = df_clean.group_by("hour").agg(
    pl.count("user_id").alias("pv_count"),
    pl.col("behavior_type").filter(pl.col("behavior_type") == "buy").count().alias("buy_count")
).sort("hour")

print(hourly_activity)

# 按星期分析用户活跃度
weekday_activity = df_clean.group_by("weekday").agg(
    pl.count("user_id").alias("pv_count"),
    pl.col("behavior_type").filter(pl.col("behavior_type") == "buy").count().alias("buy_count")
).sort("weekday")

print(weekday_activity)

# 计算RFM指标
current_date = df_clean.select(pl.col("date").max()).item()

rfm = df_clean.group_by("user_id").agg(
    # R：最近一次购买距离今天的天数
    (current_date - pl.col("date").filter(pl.col("behavior_type") == "buy").max()).dt.days().alias("recency"),
    # F：购买频率
    pl.col("behavior_type").filter(pl.col("behavior_type") == "buy").count().alias("frequency"),
    # M：购买总金额
    (pl.col("price") * (pl.col("behavior_type") == "buy")).sum().alias("monetary")
).filter(pl.col("frequency") > 0)  # 只保留有购买行为的用户

# RFM评分（1-5分，越高越好）
rfm = rfm.with_columns(
    pl.col("recency").qcut(5, labels=[5,4,3,2,1]).alias("r_score"),
    pl.col("frequency").qcut(5, labels=[1,2,3,4,5]).alias("f_score"),
    pl.col("monetary").qcut(5, labels=[1,2,3,4,5]).alias("m_score")
)

# 计算RFM总得分
rfm = rfm.with_columns(
    (pl.col("r_score").cast(int) + pl.col("f_score").cast(int) + pl.col("m_score").cast(int)).alias("rfm_score")
)

# 用户分层
def classify_user(row):
    if row["r_score"] >=4 and row["f_score"] >=4 and row["m_score"] >=4:
        return "高价值用户"
    elif row["r_score"] >=4 and row["f_score"] >=2 and row["m_score"] >=2:
        return "潜力用户"
    elif row["r_score"] <=2 and row["f_score"] >=4 and row["m_score"] >=4:
        return "召回用户"
    elif row["r_score"] <=2 and row["f_score"] <=2 and row["m_score"] <=2:
        return "流失用户"
    else:
        return "一般用户"

rfm = rfm.with_columns(
    pl.struct(["r_score", "f_score", "m_score"]).apply(classify_user).alias("user_segment")
)

# 统计各层级用户数量
segment_stats = rfm.group_by("user_segment").agg(
    pl.count("user_id").alias("count"),
    pl.sum("monetary").alias("total_revenue")
).sort("total_revenue", descending=True)

print(segment_stats)

import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots

# 1. 行为类型分布饼图
fig1 = px.pie(
    behavior_stats.to_pandas(), 
    values='count', 
    names='behavior_type',
    title='用户行为类型分布',
    labels={'behavior_type': '行为类型', 'count': '数量'},
    hole=0.3
)
fig1.show()

# 2. 小时活跃度折线图
hourly_pd = hourly_activity.to_pandas()
fig2 = make_subplots(specs=[[{"secondary_y": True}]])
fig2.add_trace(
    go.Scatter(x=hourly_pd['hour'], y=hourly_pd['pv_count'], name='浏览量'),
    secondary_y=False
)
fig2.add_trace(
    go.Scatter(x=hourly_pd['hour'], y=hourly_pd['buy_count'], name='购买量'),
    secondary_y=True
)
fig2.update_layout(title='用户活跃度小时分布', xaxis_title='小时')
fig2.update_yaxes(title_text='浏览量', secondary_y=False)
fig2.update_yaxes(title_text='购买量', secondary_y=True)
fig2.show()

# 3. 用户分层柱状图
segment_pd = segment_stats.to_pandas()
fig3 = px.bar(
    segment_pd,
    x='user_segment',
    y='count',
    color='user_segment',
    text='count',
    title='用户分层分布',
    labels={'user_segment': '用户层级', 'count': '用户数量'}
)
fig3.show()

# 4. RFM得分散点图
fig4 = px.scatter_3d(
    rfm.to_pandas(),
    x='r_score',
    y='f_score',
    z='m_score',
    color='user_segment',
    title='RFM用户3D分布',
    labels={'r_score': '最近购买', 'f_score': '购买频率', 'm_score': '消费金额'}
)
fig4.show()