目次
pandas使うのになぜか毎回勉強し直してるため
sample
import pandas as pd
data = { "name": ["Tanaka", "Sato", "Suzuki", "Takahashi", "Ito"], "dept": ["Sales", "Sales", "Dev", "Dev", "HR"], "age": [34, 28, 41, 25, 38], "salary": [320, 280, 410, 250, None],}df = pd.DataFrame(data)pd.read_csv("data.csv"), pd.read_parquet("data.parquet")でcsv, parquetからも読み込み可能
以下この適当なダミーデータを使います
基本情報
df.head()# name dept age salary# 0 Tanaka Sales 34 320.0# 1 Sato Sales 28 280.0# 2 Suzuki Dev 41 410.0# 3 Takahashi Dev 25 250.0# 4 Ito HR 38 NaN
df.info()# <class 'pandas.core.frame.DataFrame'># RangeIndex: 5 entries, 0 to 4# Data columns (total 4 columns):# # Column Non-Null Count Dtype# --- ------ -------------- -----# 0 name 5 non-null object# 1 dept 5 non-null object# 2 age 5 non-null int64# 3 salary 4 non-null float64# dtypes: float64(1), int64(1), object(2)# memory usage: 292.0+ bytes
df.shape# (5, 4)
# row x column
df.dtypes# name object# dept object# age int64# salary float64# dtype: object
df.describe()# age salary# count 5.000000 4.000000# mean 33.200000 315.000000# std 6.685806 69.522179# min 25.000000 250.000000# 25% 28.000000 272.500000# 50% 34.000000 300.000000# 75% 38.000000 342.500000# max 41.000000 410.000000
# 数値列の統計量を出力、文字列列は自動で除外選択
df[df["age"] > 30]# name dept age salary# 0 Tanaka Sales 34 320.0# 2 Suzuki Dev 41 410.0# 4 Ito HR 38 NaN
df[(df["age"] > 30) & (df["dept"] == "Dev")] # 30歳以上且つdev所属# name dept age salary# 2 Suzuki Dev 41 410.0
df[df["dept"].isin(["Sales", "Dev"])] # deptがsales, devが含まれる, isin() -> boolean# name dept age salary# 0 Tanaka Sales 34 320.0# 1 Sato Sales 28 280.0# 2 Suzuki Dev 41 410.0# 3 Takahashi Dev 25 250.0
df.loc[df["age"] > 30, ["name", "age"]] # age30以上かつ列をname, ageに絞る# name age# 0 Tanaka 34# 2 Suzuki 41# 4 Ito 38
df.loc[0:2] # [row, column]# name dept age salary# 0 Tanaka Sales 34 320.0# 1 Sato Sales 28 280.0# 2 Suzuki Dev 41 410.0
# pythonスライスは0, 1で2は含まないが、pandasは両端を含む0, 1, 2
df.iloc[0:2, 0:2]# name dept# 0 Tanaka Sales# 1 Sato Sales
# ilocはpythonスライスと同様、第1引数が行、2が列指定
df.query("age > 30 and salary > 300")# name dept age salary# 0 Tanaka Sales 34 320.0# 2 Suzuki Dev 41 410.0
# 詳しいクエリ:# https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.query.html
df.nlargest(3, "salary")# name dept age salary# 2 Suzuki Dev 41 410.0# 0 Tanaka Sales 34 320.0# 1 Sato Sales 28 280.0
# salaryからlargest3つ
df.nsmallest(2, "age")# name dept age salary# 3 Takahashi Dev 25 250.0# 1 Sato Sales 28 280.0
df.sample(2)# name dept age salary# 2 Suzuki Dev 41 410.0# 0 Tanaka Sales 34 320.0追加
df["bonus"] = df["salary"] * 0.1# name dept age salary bonus# 0 Tanaka Sales 34 320.0 32.0# 1 Sato Sales 28 280.0 28.0# 2 Suzuki Dev 41 410.0 41.0# 3 Takahashi Dev 25 250.0 25.0# 4 Ito HR 38 NaN NaN
# 以下、dfに対する破壊操作
df["is_senior"] = df["age"] >= 35# name dept age salary bonus is_senior# 0 Tanaka Sales 34 320.0 32.0 False# 1 Sato Sales 28 280.0 28.0 False# 2 Suzuki Dev 41 410.0 41.0 True# 3 Takahashi Dev 25 250.0 25.0 False# 4 Ito HR 38 NaN NaN True
df["level"] = df["age"].apply(lambda x: "senior" if x >= 35 else "junior")# name dept age salary bonus is_senior level# 0 Tanaka Sales 34 320.0 32.0 False junior# 1 Sato Sales 28 280.0 28.0 False junior# 2 Suzuki Dev 41 410.0 41.0 True senior# 3 Takahashi Dev 25 250.0 25.0 False junior# 4 Ito HR 38 NaN NaN True senior
# 内部で行ループに近い処理をするため、非効率# 大規模データには非推奨
df["full_info"] = df["name"] + " (" + df["dept"] + ")"# name dept age salary bonus is_senior level full_info# 0 Tanaka Sales 34 320.0 32.0 False junior Tanaka (Sales)# 1 Sato Sales 28 280.0 28.0 False junior Sato (Sales)# 2 Suzuki Dev 41 410.0 41.0 True senior Suzuki (Dev)# 3 Takahashi Dev 25 250.0 25.0 False junior Takahashi (Dev)# 4 Ito HR 38 NaN NaN True senior Ito (HR)
df.drop("bonus", axis=1)# name dept age salary is_senior level full_info# 0 Tanaka Sales 34 320.0 False junior Tanaka (Sales)# 1 Sato Sales 28 280.0 False junior Sato (Sales)# 2 Suzuki Dev 41 410.0 True senior Suzuki (Dev)# 3 Takahashi Dev 25 250.0 False junior Takahashi (Dev)# 4 Ito HR 38 NaN True senior Ito (HR)
# 非破壊操作, inplace=Trueで破壊操作かつ戻り値無し# axis=1で列に対して、0で行に対して
df.drop(columns=["bonus", "level"])# name dept age salary is_senior full_info# 0 Tanaka Sales 34 320.0 False Tanaka (Sales)# 1 Sato Sales 28 280.0 False Sato (Sales)# 2 Suzuki Dev 41 410.0 True Suzuki (Dev)# 3 Takahashi Dev 25 250.0 False Takahashi (Dev)# 4 Ito HR 38 NaN True Ito (HR)
df.drop(0, axis=0)# name dept age salary bonus is_senior level full_info# 1 Sato Sales 28 280.0 28.0 False junior Sato (Sales)# 2 Suzuki Dev 41 410.0 41.0 True senior Suzuki (Dev)# 3 Takahashi Dev 25 250.0 25.0 False junior Takahashi (Dev)# 4 Ito HR 38 NaN NaN True senior Ito (HR)
df.drop(index=[0, 1])# name dept age salary bonus is_senior level full_info# 2 Suzuki Dev 41 410.0 41.0 True senior Suzuki (Dev)# 3 Takahashi Dev 25 250.0 25.0 False junior Takahashi (Dev)# 4 Ito HR 38 NaN NaN True senior Ito (HR)
df.rename(columns={"dept": "department"})# name department age salary bonus is_senior level full_info# 0 Tanaka Sales 34 320.0 32.0 False junior Tanaka (Sales)# 1 Sato Sales 28 280.0 28.0 False junior Sato (Sales)# 2 Suzuki Dev 41 410.0 41.0 True senior Suzuki (Dev)# 3 Takahashi Dev 25 250.0 25.0 False junior Takahashi (Dev)# 4 Ito HR 38 NaN NaN True senior Ito (HR)
# 非破壊操作
df.insert(1, "id", range(1, len(df) + 1))# name id dept age salary bonus is_senior level full_info# 0 Tanaka 1 Sales 34 320.0 32.0 False junior Tanaka (Sales)# 1 Sato 2 Sales 28 280.0 28.0 False junior Sato (Sales)# 2 Suzuki 3 Dev 41 410.0 41.0 True senior Suzuki (Dev)# 3 Takahashi 4 Dev 25 250.0 25.0 False junior Takahashi (Dev)# 4 Ito 5 HR 38 NaN NaN True senior Ito (HR)
# 列インデックス1（つまり2列目）にidを挿入、1~行数分の連番を追加# 破壊操作欠損値
df.isna()# name id dept age salary bonus is_senior level full_info# 0 False False False False False False False False False# 1 False False False False False False False False False# 2 False False False False False False False False False# 3 False False False False False False False False False# 4 False False False False True True False False False
# 欠損してるとこだけTrue
df.isna().sum()# name 0# id 0# dept 0# age 0# salary 1# bonus 1# is_senior 0# level 0# full_info 0# dtype: int64
df.isna().any()# name False# id False# dept False# age False# salary True# bonus True# is_senior False# level False# full_info False# dtype: bool
df.dropna()# name id dept age salary bonus is_senior level full_info# 0 Tanaka 1 Sales 34 320.0 32.0 False junior Tanaka (Sales)# 1 Sato 2 Sales 28 280.0 28.0 False junior Sato (Sales)# 2 Suzuki 3 Dev 41 410.0 41.0 True senior Suzuki (Dev)# 3 Takahashi 4 Dev 25 250.0 25.0 False junior Takahashi (Dev)
# NaNがある行を削除# 非破壊
df.dropna(subset=["salary"])# dropna()をsalaryに限定する
df.fillna(0)# name id dept age salary bonus is_senior level full_info# 0 Tanaka 1 Sales 34 320.0 32.0 False junior Tanaka (Sales)# 1 Sato 2 Sales 28 280.0 28.0 False junior Sato (Sales)# 2 Suzuki 3 Dev 41 410.0 41.0 True senior Suzuki (Dev)# 3 Takahashi 4 Dev 25 250.0 25.0 False junior Takahashi (Dev)# 4 Ito 5 HR 38 0.0 0.0 True senior Ito (HR)
df.fillna({"salary": 0, "bonus": df["bonus"].mean()})# name dept age salary bonus is_senior level full_info# 0 Tanaka Sales 34 320.0 32.0 False junior Tanaka (Sales)# 1 Sato Sales 28 280.0 28.0 False junior Sato (Sales)# 2 Suzuki Dev 41 410.0 41.0 True senior Suzuki (Dev)# 3 Takahashi Dev 25 250.0 25.0 False junior Takahashi (Dev)# 4 Ito HR 38 0.0 31.5 True senior Ito (HR)
# salaryは0埋め、bonusはbonus列の平均で埋め
df["salary"].fillna(df["salary"].mean())# 0 320.0# 1 280.0# 2 410.0# 3 250.0# 4 315.0# Name: salary, dtype: float64
# 一列だけの抽出
df.ffill()# name dept age salary bonus is_senior level full_info# 0 Tanaka Sales 34 320.0 32.0 False junior Tanaka (Sales)# 1 Sato Sales 28 280.0 28.0 False junior Sato (Sales)# 2 Suzuki Dev 41 410.0 41.0 True senior Suzuki (Dev)# 3 Takahashi Dev 25 250.0 25.0 False junior Takahashi (Dev)# 4 Ito HR 38 250.0 25.0 True senior Ito (HR)
# forward fill, 前の値で埋める 逆はdf.bfill()ソート
df.sort_values("salary", ascending=False)# name dept age salary bonus is_senior level full_info# 2 Suzuki Dev 41 410.0 41.0 True senior Suzuki (Dev)# 0 Tanaka Sales 34 320.0 32.0 False junior Tanaka (Sales)# 1 Sato Sales 28 280.0 28.0 False junior Sato (Sales)# 3 Takahashi Dev 25 250.0 25.0 False junior Takahashi (Dev)# 4 Ito HR 38 NaN NaN True senior Ito (HR)
# salary列でソート, ascending=Falseで降順, Trueで昇順
df.sort_values(["dept", "age"], ascending=[True, False])# name dept age salary bonus is_senior level full_info# 2 Suzuki Dev 41 410.0 41.0 True senior Suzuki (Dev)# 3 Takahashi Dev 25 250.0 25.0 False junior Takahashi (Dev)# 4 Ito HR 38 NaN NaN True senior Ito (HR)# 0 Tanaka Sales 34 320.0 32.0 False junior Tanaka (Sales)# 1 Sato Sales 28 280.0 28.0 False junior Sato (Sales)
# 2段階でのソート、deptのアルファベット順のあとageの年齢が高い順
df.sort_values("salary", na_position="first")# name dept age salary bonus is_senior level full_info# 4 Ito HR 38 NaN NaN True senior Ito (HR)# 3 Takahashi Dev 25 250.0 25.0 False junior Takahashi (Dev)# 1 Sato Sales 28 280.0 28.0 False junior Sato (Sales)# 0 Tanaka Sales 34 320.0 32.0 False junior Tanaka (Sales)# 2 Suzuki Dev 41 410.0 41.0 True senior Suzuki (Dev)
# NaNを先頭に持っていく引数、デフォは最後
df.sort_index()# name dept age salary bonus is_senior level full_info# 0 Tanaka Sales 34 320.0 32.0 False junior Tanaka (Sales)# 1 Sato Sales 28 280.0 28.0 False junior Sato (Sales)# 2 Suzuki Dev 41 410.0 41.0 True senior Suzuki (Dev)# 3 Takahashi Dev 25 250.0 25.0 False junior Takahashi (Dev)# 4 Ito HR 38 NaN NaN True senior Ito (HR)
# 元の順番、インデックスを戻す
df.sort_values("salary").reset_index(drop=True)# name dept age salary bonus is_senior level full_info# 0 Takahashi Dev 25 250.0 25.0 False junior Takahashi (Dev)# 1 Sato Sales 28 280.0 28.0 False junior Sato (Sales)# 2 Tanaka Sales 34 320.0 32.0 False junior Tanaka (Sales)# 3 Suzuki Dev 41 410.0 41.0 True senior Suzuki (Dev)# 4 Ito HR 38 NaN NaN True senior Ito (HR)
# salaryで並び替えたのちインデックスをリセット# drop=Trueにしないと旧インデックスが列として残る統計
df.groupby("dept")["salary"].mean()# dept# Dev 330.0# HR NaN# Sales 300.0# Name: salary, dtype: float64
# deptごとのグループでsalaryの平均を出す# sum()は同様に合計
df.groupby("dept").agg({"salary": "mean", "age": "max"})# salary age# dept# Dev 330.0 41# HR NaN 38# Sales 300.0 34
# 列ごとに違う方法での統計を行える# agg = aggregate
df.groupby("dept").agg(avg_salary=("salary", "mean"), count=("name", "count"))# avg_salary count# dept# Dev 330.0 2# HR NaN 1# Sales 300.0 2
# agg(新しい列名=("集計対象列", "統計方法"), ...)
df.groupby("dept").size()# dept# Dev 2# HR 1# Sales 2# dtype: int64
# countは非NaN要素, size()は全てtips
loopできる、ケドあんまりしない方がいい
for row in df.itertuples(): print(row.Index, row.name, row.salary)全体的にベクトル演算を優先した方がいい、とのこと