翻訳待ち：From In-Silico to Wet-Lab: Evaluating AI Protein Design Performance

AI サービスが一時的に利用できないため、復旧後に翻訳を補完します。

In this tutorial, we use Anthropic’s claude-protein-binder-design dataset, which contains 1,440 AI-designed miniprotein binders tested against 16 targets. Because the release includes both computational predictions and real wet-lab results from two independent labs, we can go beyond simply studying the designs. We evaluate how well structure predictors identify successful binders, whether combining predictions improves performance, how rankings translate into practical testing budgets, and how much disagreement comes from the assays themselves. Also, we train a target-aware classifier to test whether these signals can reliably predict experimental success. Copy CodeCopiedUse a different Browser import subprocess, sys, warnings, itertools, math warnings.filterwarnings("ignore") import importlib.util _needed = {"huggingface_hub": "huggingface_hub>=0.24", "pyarrow": "pyarrow", "pandas": "pandas", "sklearn": "scikit-learn", "matplotlib": "matplotlib", "scipy": "scipy"} _missing = [pkg for mod, pkg in _needed.items() if importlib.util.find_spec(mod) is None] if _missing: print("installing:", ", ".join(_missing)) subprocess.run([sys.executable, "-m", "pip", "install", "-q", *_missing], check=False) import numpy as np import pandas as pd import matplotlib.pyplot as plt from scipy import stats from huggingface_hub import HfApi, hf_hub_download from sklearn.metrics import roc_auc_score, cohen_kappa_score, average_precision_score from sklearn.model_selection import GroupKFold, StratifiedKFold from sklearn.ensemble import HistGradientBoostingClassifier from sklearn.inspection import permutation_importance SEED = 0 rng_global = np.random.default_rng(SEED) pd.set_option("display.width", 200) pd.set_option("display.max_columns", 100) plt.rcParams.update({"figure.dpi": 110, "font.size": 9, "axes.grid": True, "grid.alpha": 0.25, "axes.spines.top": False, "axes.spines.right": False}) REPO = "Anthropic/claude-protein-binder-design" BAR = "=" * 78 def head(n, title): prefix = f"{n}. " if str(n) else "" print(f"\n{BAR}\n {prefix}{title}\n{BAR}") head(1, "TABLE DISCOVERY") api = HfApi() repo_files = api.list_repo_files(REPO, repo_type="dataset") TABLES = {} for f in repo_files: if f.startswith("data/tables/") and f.endswith(".parquet"): key = f[len("data/tables/"): -len(".parquet")].replace("/", "_") TABLES[key] = f print(f"Found {len(TABLES)} Parquet tables:") for k in sorted(TABLES): print(f" - {k:38s} {TABLES[k]}") def load_table(name: str) -> pd.DataFrame: """Load a subset by its viewer name, with a datasets-library fallback.""" if name in TABLES: return pd.read_parquet(hf_hub_download(REPO, TABLES[name], repo_type="dataset")) from datasets import load_dataset return load_dataset(REPO, name, split="full").to_pandas() ds = load_table("design_summary") print(f"\ndesign_summary: {ds.shape[0]:,} rows x {ds.shape[1]} columns") We start by installing only what the runtime is actually missing, then enumerate the repository once and build a {subset to path} map instead of hard-coding file locations. This matters because the naming is not uniform; the subset wetlab_summary lives at data/tables/wetlab/summary.parquet, and a guessed path would fail silently. With the map in place we pull design_summary, one row per design, 1,440 rows wide enough to carry every join we need downstream. Copy CodeCopiedUse a different Browser head(2, "SCHEMA + EVALUABLE SET") CALLS = {"binder", "non_binder"} tested = ds["adaptyv_binding"].isin(CALLS) | ds["twist_binding"].isin(CALLS) ev = ds[tested].copy() ev["y"] = ev["binder_final"].astype(int) print(f"All designs : {len(ds):,}") print(f"Evaluable (>=1 vendor call): {len(ev):,}") print(f"Confirmed binders : {int(ev['y'].sum()):,} " f"({100 * ev['y'].mean():.1f}% base rate)") print(f"Never measured : {len(ds) - len(ev):,}") print("\nCategorical levels:") for c in ["design_model", "campaign", "generator", "sequence_design_method", "vendor_agreement"]: vals = ds[c].astype(str).value_counts() print(f" {c:24s} ({len(vals)}): {', '.join(vals.index[:6])}" + (" ..." if len(vals) > 6 else "")) print(f"\nTargets ({ds['target'].nunique()}): {', '.join(sorted(ds['target'].unique()))}") print(f"Binder length: {ds.binder_length.min()}-{ds.binder_length.max()} aa " f"(median {ds.binder_length.median():.0f})") head(3, "HIT-RATE LANDSCAPE") def wilson(k, n, z=1.96): if n == 0: return (np.nan, np.nan, np.nan) p = k / n d = 1 + z2 / n c = (p + z2 / (2 * n)) / d h = z * math.sqrt(p * (1 - p) / n + z2 / (4 * n2)) / d return p, max(0.0, c - h), min(1.0, c + h) def rate_table(df, by): rows = [] for key, g in df.groupby(by, dropna=False): p, lo, hi = wilson(int(g.y.sum()), len(g)) rows.append({by: key, "n": len(g), "hits": int(g.y.sum()), "rate": p, "lo": lo, "hi": hi}) return pd.DataFrame(rows).sort_values("rate", ascending=False).reset_index(drop=True) for dim in ["design_model", "campaign", "generator", "sequence_design_method"]: t = rate_table(ev, dim) print(f"\n--- hit rate by {dim} ---") print(t.to_string(index=False, formatters={"rate": "{:.3f}".format, "lo": "{:.3f}".format, "hi": "{:.3f}".format})) tt = rate_table(ev, "target") fig, ax = plt.subplots(figsize=(9, 4.2)) ax.bar(tt.target, tt.rate, color="#4C72B0") ax.errorbar(tt.target, tt.rate, yerr=[(tt.rate - tt.lo).clip(lower=0), (tt.hi - tt.rate).clip(lower=0)], fmt="none", ecolor="0.25", capsize=3, lw=1) ax.axhline(ev.y.mean(), ls="--", c="crimson", lw=1, label=f"pooled {ev.y.mean():.2f}") ax.set_ylabel("experimental hit rate"); ax.set_title("Hit rate by target (Wilson 95% CI)") ax.tick_params(axis="x", rotation=55); ax.legend(); plt.tight_layout(); plt.show() print("\nRead this plot as the dominant effect size in the dataset: target choice " "swamps generator choice. Any model comparison that does not stratify by " "target is mostly measuring which targets that model was pointed at.") We define the evaluable set by filtering on actual vendor calls rather than on binder_final, because that column is a bool and so records the 120 never-measured designs as False rather than missing. From there we compute hit rates by model, campaign, generator, and target, wrapping each in a Wilson interval since several subgroups sit in the small-n regime where the normal approximation misbehaves. The target plot is the one to read first: it shows antigen choice swamping every other factor we compare. Copy CodeCopiedUse a different Browser head(4, "PER-PREDICTOR DISCRIMINATIVE POWER") PREDICTORS = sorted({c[len("ipsae_min_"):] for c in ds.columns if c.startswith("ipsae_min_")}) print(f"Predictors ({len(PREDICTORS)}): {', '.join(PREDICTORS)}") def auc_ci(y, s, n_boot=300, seed=SEED): s = np.asarray(s, dtype=float); y = np.asarray(y, dtype=int) m = ~np.isnan(s) y, s = y[m], s[m] if len(y) 1: boots.append(roc_auc_score(y[b], s[b])) lo, hi = (np.percentile(boots, [2.5, 97.5]) if boots else (np.nan, np.nan)) return dict(auc=base, lo=lo, hi=hi, n=len(y), ap=ap) rows = [] for p in PREDICTORS: for metric in ["ipsae_min", "sc_dockq"]: col = f"{metric}_{p}" if col in ev.columns: r = auc_ci(ev.y, ev[col]) rows.append({"predictor": p, "metric": metric, **r}) perf = pd.DataFrame(rows) piv = perf.pivot(index="predictor", columns="metric", values="auc").sort_values("ipsae_min", ascending=False) print("\nAUC vs experimental binder_final:") print(perf.sort_values("auc", ascending=False).to_string( index=False, formatters={c: "{:.3f}".format for c in ["auc", "lo", "hi", "ap"]})) fig, ax = plt.subplots(figsize=(9, 4.2)) x = np.arange(len(piv)); w = 0.38 for i, (metric, colr) in enumerate([("ipsae_min", "#4C72B0"), ("sc_dockq", "#DD8452")]): sub = perf[perf.metric == metric].set_index("predictor").reindex(piv.index) lo_err = (sub.auc - sub.lo).clip(lower=0).fillna(0) hi_err = (sub.hi - sub.auc).clip(lower=0).fillna(0) ax.bar(x + (i - 0.5) * w, sub.auc, w, label=metric, color=colr) ax.errorbar(x + (i - 0.5) * w, sub.auc, yerr=[lo_err, hi_err], fmt="none", ecolor="0.3", capsize=2, lw=0.9) ax.axhline(0.5, ls="--", c="crimson", lw=1) ax.set_xticks(x); ax.set_xticklabels(piv.index, rotation=45, ha="right") ax.set_ylabel("AUC"); ax.set_ylim(0.35, None) ax.set_title("In-silico score vs wet-lab binding, by structure predictor") ax.legend(); plt.tight_layout(); plt.show() print("Interpretation: AUCs land well above chance but far below the ~0.9 you " "would need to trust a single filter. That gap is the entire practical " "reason this dataset exists.") head(5, "CONSENSUS SCORING") ips_cols = [f"ipsae_min_{p}" for p in PREDICTORS if f"ipsae_min_{p}" in ev.columns] dq_cols = [f"sc_dockq_{p}" for p in PREDICTORS if f"sc_dockq_{p}" in ev.columns] def pct_rank(df, cols): return df[cols].rank(pct=True, na_option="keep") R_ips, R_dq = pct_rank(ev, ips_cols), pct_rank(ev, dq_cols) ev["cons_ipsae"] = R_ips.mean(axis=1) ev["cons_dockq"] = R_dq.mean(axis=1) ev["cons_all"] = pd.concat([R_ips, R_dq], axis=1).mean(axis=1) ev["cons_median"] = pd.concat([R_ips, R_dq], axis=1).median(axis=1) ev["cons_min"] = pd.concat([R_ips, R_dq], axis=1).min(axis=1) ev["cons_disagree"] = pd.concat([R_ips, R_dq], axis=1).std(axis=1) best_single = perf.loc[perf.auc.idxmax()] print(f"Best single column: {best_single.metric}_{best_single.predictor} AUC={best_single.auc:.3f}") print() for name in ["cons_ipsae", "cons_dockq", "cons_all", "cons_median", "cons_min", "cons_disagree"]: r = auc_ci(ev.y, ev[name]) print(f" {name:16s} AUC={r['auc']:.3f} [{r['lo']:.3f}, {r['hi']:.3f}] AP={r['ap']:.3f}") corr = ev[ips_cols].corr(method="spearman") fig, ax = plt.subplots(figsize=(6.2, 5.2)) im = ax.imshow(corr.values, cmap="viridis", vmin=0, vmax=1) lbl = [c.replace("ipsae_min_", "") for c in ips_cols] ax.set_xticks(range(len(lbl))); ax.set_xticklabels(lbl, rotation=90) ax.set_yticks(range(len(lbl))); ax.set_yticklabels(lbl) ax.set_title("Spearman correlation between predictors (ipSAE)") ax.grid(False); fig.colorbar(im, shrink=0.8); plt.tight_layout(); plt.show() print("\nIf every off-diagonal cell were ~1.0 there would be no ensemble gain to " "harvest. The moderate correlations are why cons_all typically edges out " "the best single predictor — and why disagreement itself carries signal.") We score all ten predictors against the wet-lab label, on both ipSAE and self-consistency DockQ, with bootstrapped confidence intervals so we can see which differences are real. We then rank-normalize each column to percentiles and aggregate them, which keeps the comparison scale-free across metrics that live on different ranges and pile up differently at zero. The Spearman heatmap explains why the ensemble helps at all; if the predictors agreed perfectly there would be nothing left to harvest. Copy CodeCopiedUse a different Browser head(6, "BUDGET CURVES (precision@N)") def budget_curve(df, score_col, max_n=400): d = df[[score_col, "y"]].dropna().sort_values(score_col, ascending=False) hits = d.y.values.cumsum() n = np.arange(1, len(d) + 1) k = min(max_n, len(d)) return n[:k], (hits / n)[:k] fig, ax = plt.subplots(figsize=(8, 4.4)) best_col = f"{best_single.metric}_{best_single.predictor}" for col, lab, style in [(best_col, f"best single ({best_col})", "-"), ("cons_all", "consensus (rank-avg, all)", "-"), ("cons_min", "consensus (unanimity/min)", "--")]: n, prec = budget_curve(ev, col) ax.plot(n, prec, style, lw=1.8, label=lab) ax.axhline(ev.y.mean(), ls=":", c="crimson", lw=1.4, label=f"random baseline ({ev.y.mean():.2f})") ax.set_xlabel("designs ordered for wet-lab testing (N, best-first)") ax.set_ylabel("hit rate among top N"); ax.set_title("How much does in-silico triage buy you?") ax.legend(); plt.tight_layout(); plt.show() print("Enrichment at small budgets:") for N in [25, 50, 100, 200]: line = f" N={N:4d} | random {ev.y.mean():.3f}" for col, lab in [(best_col, "best-single"), ("cons_all", "consensus")]: n, prec = budget_curve(ev, col, max_n=N) line += f" | {lab} {prec[-1]:.3f} ({prec[-1] / ev.y.mean():.2f}x)" print(line) head(7, "VENDOR CONCORDANCE") both = ev[ev.adaptyv_ [truncated for AI cost control]