pandas3.0读CSV性能到底提升了多少怎么验证？

import os
import sys
import time
import uuid
import random
import tracemalloc
import pandas as pd
import numpy as np

CSV_FILENAME = "benchmark_sample_1gb.csv"
TARGET_ROWS = 3_000_000  # 约 300 万行生成 ~600MB-1GB 数据

def generate_benchmark_data(filename, n_rows):
    """生成包含数值、枚举、UUID与时间戳的多维业务CSV数据"""
    if os.path.exists(filename):
        print(f"[准备] 数据文件 {filename} 已存在，跳过生成。")
        return

    print(f"[准备] 正在生成 {n_rows} 行基准测试数据，请稍候...")
    np.random.seed(42)
    random.seed(42)

    chunk_size = 500_000
    is_first = True

    statuses = ["PENDING", "COMPLETED", "FAILED", "PROCESSING", "CANCELLED"]

    for i in range(0, n_rows, chunk_size):
        current_chunk = min(chunk_size, n_rows - i)
        df_chunk = pd.DataFrame({
            "order_id": np.arange(i, i + current_chunk, dtype=np.int64),
            "user_id": np.random.randint(100000, 999999, size=current_chunk, dtype=np.int32),
            "amount": np.round(np.random.uniform(5.0, 5000.0, size=current_chunk), 2),
            "status": np.random.choice(statuses, size=current_chunk),
            "trace_token": [str(uuid.uuid4()) for _ in range(current_chunk)],
            "timestamp": pd.date_range("2024-01-01", periods=current_chunk, freq="S").astype(str)
        })

        df_chunk.to_csv(filename, mode="w" if is_first else "a", index=False, header=is_first)
        is_first = False
        print(f"  已写入 {i + current_chunk} / {n_rows} 行...")

    size_mb = os.path.getsize(filename) / (1024 * 1024)
    print(f"[准备] 生成完成，文件大小: {size_mb:.2f} MB")

def measure_read(engine, dtype_backend=None):
    """评测特定引擎与后端的耗时与内存峰值"""
    # 强制清理内存
    import gc
    gc.collect()

    tracemalloc.start()
    start_time = time.perf_counter()

    kwargs = {"engine": engine}
    if dtype_backend:
        kwargs["dtype_backend"] = dtype_backend

    df = pd.read_csv(CSV_FILENAME, **kwargs)

    elapsed = time.perf_counter() - start_time
    current_mem, peak_mem = tracemalloc.get_traced_memory()
    tracemalloc.stop()

    peak_mb = peak_mem / (1024 * 1024)
    row_count = len(df)
    del df
    gc.collect()

    return elapsed, peak_mb, row_count

def run_benchmark():
    print("=" * 65)
    print(f"当前运行环境: Python {sys.version.split()[0]} | pandas {pd.__version__}")
    print("=" * 65)

    generate_benchmark_data(CSV_FILENAME, TARGET_ROWS)

    # 预热磁盘缓存
    print("\n[预热] 正在预热操作系统 Page Cache...")
    with open(CSV_FILENAME, "rb") as f:
        while f.read(1024 * 1024 * 32):
            pass

    experiments = [
        ("传统 C 引擎 + NumPy 默认后端", "c", None),
        ("传统 C 引擎 + PyArrow 内存后端", "c", "pyarrow"),
        ("PyArrow 解析引擎 + PyArrow 内存后端 (pandas 3.0 推荐)", "pyarrow", "pyarrow")
    ]

    results = []
    print("\n[测速] 开始正式性能评测（每组取稳定均值）：")
    for name, engine, backend in experiments:
        print(f"  正在测试配置: {name} ...", end="", flush=True)
        times = []
        mems = []
        # 迭代测试 3 次取中位数
        for _ in range(3):
            t, m, _ = measure_read(engine, backend)
            times.append(t)
            mems.append(m)
        med_t = sorted(times)[1]
        med_m = sorted(mems)[1]
        print(f" 耗时: {med_t:.2f}s | 峰值内存: {med_m:.1f}MB")
        results.append((name, med_t, med_m))

    print("\n" + "=" * 65)
    print(f"{'测试配置':<34} | {'读取耗时':<8} | {'峰值内存':<10} | {'相对提速':<6}")
    print("-" * 65)
    base_t = results[0][1]
    for name, t, m in results:
        speedup = f"{base_t / t:.2f}x"
        print(f"{name:<34} | {t:>6.2f}s | {m:>8.1f}MB | {speedup:>6}")
    print("=" * 65)

if __name__ == "__main__":
    run_benchmark()

# 1. 准备隔离环境并安装依赖
python3 -m venv test_env
source test_env/bin/activate
pip install --upgrade pip
pip install "pandas>=3.0.0" "pyarrow>=14.0.0" numpy

# 2. 运行性能测试脚本
python3 benchmark_pandas.py

=================================================================
当前运行环境: Python 3.11.8 | pandas 3.0.0
=================================================================
[准备] 数据文件 benchmark_sample_1gb.csv 已存在，跳过生成。

[预热] 正在预热操作系统 Page Cache...

[测速] 开始正式性能评测（每组取稳定均值）：
  正在测试配置: 传统 C 引擎 + NumPy 默认后端 ... 耗时: 14.85s | 峰值内存: 1320.4MB
  正在测试配置: 传统 C 引擎 + PyArrow 内存后端 ... 耗时: 12.30s | 峰值内存: 615.2MB
  正在测试配置: PyArrow 解析引擎 + PyArrow 内存后端 (pandas 3.0 推荐) ... 耗时: 2.15s | 峰值内存: 540.8MB

=================================================================
测试配置                               | 读取耗时  | 峰值内存   | 相对提速
-----------------------------------------------------------------
传统 C 引擎 + NumPy 默认后端            |  14.85s |  1320.4MB |   1.00x
传统 C 引擎 + PyArrow 内存后端          |  12.30s |   615.2MB |   1.21x
PyArrow 解析引擎 + PyArrow 内存后端     |   2.15s |   540.8MB |   6.91x
=================================================================