Optimasi read_csv()

import shutil
import tempfile
from pathlib import Path

import numpy as np
import pandas as pd

# File CSV 120.000 baris, dibangun runtime secara deterministik
rng = np.random.default_rng(42)
n = 120_000
df = pd.DataFrame(
    {
        "id_pesanan": [f"TRX-{i:06d}" for i in range(n)],
        "kota": rng.choice(["Jakarta", "Bandung", "Surabaya", "Medan", "Semarang"], size=n),
        "produk": rng.choice(
            ["Beras", "Minyak Goreng", "Gula Pasir", "Kopi", "Telur", "Tepung Terigu", "Sabun", "Sarden"],
            size=n,
        ),
        "jumlah": rng.integers(1, 11, size=n),
        "harga": rng.choice([15000, 25000, 40000, 55000, 70000], size=n),
    }
)
df["total"] = df["jumlah"] * df["harga"]
folder = tempfile.mkdtemp()
jalur = Path(folder) / "transaksi.csv"
df.to_csv(jalur, index=False)
print("file:", f"{jalur.stat().st_size:,}", "byte")
print()

# 1. Baca polos: semua kolom, dtype bawaan
dasar = pd.read_csv(jalur)
print("dtypes dasar:", dasar.dtypes.astype(str).to_dict())
print("memori dasar :", f"{int(dasar.memory_usage(deep=True).sum()):,}", "byte")
print()

# 2. usecols: baca kolom yang benar-benar dipakai saja
sedikit = pd.read_csv(jalur, usecols=["kota", "jumlah", "total"])
print("usecols kolom:", list(sedikit.columns))
print("memori usecols:", f"{int(sedikit.memory_usage(deep=True).sum()):,}", "byte")
print()

# 3. dtype eksplisit: kecilkan saat membaca, bukan sesudah
tipis = pd.read_csv(
    jalur,
    usecols=["kota", "jumlah", "total"],
    dtype={"jumlah": "int8", "kota": "category"},
)
print("dtypes tipis:", tipis.dtypes.astype(str).to_dict())
print("memori tipis :", f"{int(tipis.memory_usage(deep=True).sum()):,}", "byte")
print("rasio:", round(int(dasar.memory_usage(deep=True).sum()) / int(tipis.memory_usage(deep=True).sum()), 1))
print()

# 4. nrows: pratinjau murah sebelum membaca semua
pratinjau = pd.read_csv(jalur, nrows=5)
print("nrows=5 shape:", pratinjau.shape)
print()

# 5. Gotcha skiprows: header ikut terlewati bila salah hitung
salah = pd.read_csv(jalur, skiprows=1)
print("skiprows=1 header:", list(salah.columns)[:3])
print("skiprows=1 shape:", salah.shape)
print()

# 6. engine pyarrow: dtype identik, fitur chunksize/nrows tidak didukung
pa = pd.read_csv(jalur, engine="pyarrow", usecols=["kota", "jumlah", "total"])
print("pyarrow dtypes:", pa.dtypes.astype(str).to_dict())
print("pyarrow dtypes == engine c:", pa.dtypes.equals(dasar[["kota", "jumlah", "total"]].dtypes))
try:
    pd.read_csv(jalur, engine="pyarrow", chunksize=30_000)
except ValueError as e:
    print("pyarrow + chunksize ->", str(e).splitlines()[0])

shutil.rmtree(folder)
print("file csv sementara dihapus")

file: 5,093,605 byte

dtypes dasar: {'id_pesanan': 'str', 'kota': 'str', 'produk': 'str', 'jumlah': 'int64', 'harga': 'int64', 'total': 'int64'}
memori dasar : 8,713,123 byte

usecols kolom: ['kota', 'jumlah', 'total']
memori usecols: 3,719,184 byte

dtypes tipis: {'kota': 'category', 'jumlah': 'int8', 'total': 'int64'}
memori tipis : 1,200,380 byte
rasio: 7.3

nrows=5 shape: (5, 6)

skiprows=1 header: ['TRX-000000', 'Jakarta', 'Beras']
skiprows=1 shape: (119999, 6)

pyarrow dtypes: {'kota': 'str', 'jumlah': 'int64', 'total': 'int64'}
pyarrow dtypes == engine c: True
pyarrow + chunksize -> The 'chunksize' option is not supported with the 'pyarrow' engine
file csv sementara dihapus