"""Where the time of sklearn's permutation_importance goes on a LightGBM model, and which settings change only the time. python pi_cost.py prep WORK [TREES] synthetic data (40,000 x 50), a classifier (300 trees), saved in WORK python pi_cost.py count WORK how many predict calls one permutation_importance makes python pi_cost.py all WORK [ROUNDS [NAME...]] the settings below (or those named), each in a fresh process, ROUNDS times (default 3) python pi_cost.py run WORK NAME one timed permutation_importance Each run prints the wall time of the permutation_importance call and a hash of the full importances array (features x repeats). Equal hashes mean bit-identical results. CPUs: the ones this process may use (taskset), called N below. """ import hashlib import os import pickle import statistics import subprocess import sys import time import warnings import numpy as np ROWS_TRAIN, ROWS_EVAL, FEATURES, TREES, LEAVES, REPEATS = 30_000, 10_000, 50, 300, 63, 5 N = len(os.sched_getaffinity(0)) # name: (input, permutation_importance n_jobs, LightGBM threads, extra arguments, can change the result) SETTINGS = { "pandas, n_jobs=1, LightGBM default threads": ("pandas", 1, None, {}, False), "pandas, n_jobs=1, 1 thread": ("pandas", 1, 1, {}, False), "numpy, n_jobs=1, 1 thread": ("numpy", 1, 1, {}, False), "numpy, n_jobs=1, N threads": ("numpy", 1, N, {}, False), "numpy, n_jobs=N, 1 thread": ("numpy", N, 1, {}, False), "pandas, n_jobs=N, 1 thread": ("pandas", N, 1, {}, False), "numpy, n_jobs=N, LightGBM default threads": ("numpy", N, None, {}, False), "numpy, n_jobs=2, N/2 threads": ("numpy", 2, max(N // 2, 1), {}, False), "numpy, n_jobs=N, 1 thread, max_samples=0.5": ("numpy", N, 1, {"max_samples": 0.5}, True), "numpy, n_jobs=N, 1 thread, n_repeats=2": ("numpy", N, 1, {"n_repeats": 2}, True), "numpy float32, n_jobs=N, 1 thread": ("float32", N, 1, {}, True), } def data(): from sklearn.datasets import make_classification X, y = make_classification(n_samples=ROWS_TRAIN + ROWS_EVAL, n_features=FEATURES, n_informative=20, n_redundant=10, flip_y=0.05, random_state=0) return X[:ROWS_TRAIN], y[:ROWS_TRAIN], X[ROWS_TRAIN:], y[ROWS_TRAIN:] def prep(work, trees=TREES): import lightgbm as lgb import pandas as pd os.makedirs(work, exist_ok=True) Xt, yt, Xe, ye = data() cols = ["f%02d" % i for i in range(FEATURES)] clf = lgb.LGBMClassifier(n_estimators=trees, num_leaves=LEAVES, learning_rate=0.05, subsample=0.8, subsample_freq=1, colsample_bytree=0.8, random_state=7, deterministic=True, force_row_wise=True, verbose=-1) clf.fit(pd.DataFrame(Xt, columns=cols), yt) with open(os.path.join(work, "model.pkl"), "wb") as f: pickle.dump(clf, f) np.savez(os.path.join(work, "eval.npz"), X=Xe, y=ye) print("model: %d trees, %d leaves max; eval set %d rows x %d features" % (clf.booster_.num_trees(), LEAVES, Xe.shape[0], Xe.shape[1])) def load(work, kind): import pandas as pd with open(os.path.join(work, "model.pkl"), "rb") as f: clf = pickle.load(f) d = np.load(os.path.join(work, "eval.npz")) X, y = d["X"], d["y"] if kind == "pandas": X = pd.DataFrame(X, columns=clf.feature_name_) elif kind == "float32": X = X.astype(np.float32) return clf, X, y def run(work, name): from sklearn.inspection import permutation_importance kind, jobs, threads, extra, _ = SETTINGS[name] clf, X, y = load(work, kind) clf.set_params(n_jobs=threads) kw = dict(scoring="roc_auc", n_repeats=REPEATS, random_state=0, n_jobs=jobs) kw.update(extra) t = time.perf_counter() r = permutation_importance(clf, X, y, **kw) t = time.perf_counter() - t h = hashlib.sha256(np.ascontiguousarray(r.importances, dtype=np.float64).tobytes()).hexdigest()[:16] print("%.3f %s %.17g" % (t, h, r.importances_mean.max())) def count(work): import lightgbm as lgb from sklearn.inspection import permutation_importance calls, rows = [0], [0] original = lgb.Booster.predict def counted(self, data, *a, **k): calls[0] += 1 rows[0] += data.shape[0] return original(self, data, *a, **k) lgb.Booster.predict = counted clf, X, y = load(work, "numpy") clf.set_params(n_jobs=1) permutation_importance(clf, X, y, scoring="roc_auc", n_repeats=REPEATS, random_state=0, n_jobs=1) walks = rows[0] * clf.booster_.num_trees() print("predict calls: %d (1 + %d features x %d repeats), %d rows each, %d trees: %.0f million tree walks" % (calls[0], X.shape[1], REPEATS, rows[0] // calls[0], clf.booster_.num_trees(), walks / 1e6)) lgb.Booster.predict = original for threads in (1, N): clf.set_params(n_jobs=threads) ts = [] for _ in range(7): t = time.perf_counter() clf.predict_proba(X) ts.append(time.perf_counter() - t) print("one predict_proba over %d rows, %d thread(s): %.1f ms (median of 7)" % (X.shape[0], threads, 1e3 * statistics.median(ts))) def run_all(work, rounds, names): print("N = %d CPUs, %d rounds per setting, median wall time of the permutation_importance call" % (N, rounds)) base = None for name in names or SETTINGS: times, hashes = [], set() for _ in range(rounds): out = subprocess.run([sys.executable, "-I", os.path.abspath(__file__), "run", work, name], check=True, capture_output=True, text=True).stdout.split() times.append(float(out[0])) hashes.add(out[1]) h = hashes.pop() if len(hashes) == 1 else "varies" base = base or h print("%-48s %8.2f s %s %s" % (name, statistics.median(times), h, "same result" if h == base else "DIFFERENT result")) if __name__ == "__main__": warnings.filterwarnings("ignore") cmd, work = sys.argv[1], sys.argv[2] if cmd == "prep": prep(work, int(sys.argv[3]) if len(sys.argv) > 3 else TREES) elif cmd == "count": count(work) elif cmd == "run": run(work, sys.argv[3]) elif cmd == "all": run_all(work, int(sys.argv[3]) if len(sys.argv) > 3 else 3, sys.argv[4:])