待翻译：IMDb Sentiment Analysis with DistilBERT LoRA, TF-IDF Baselines, Calibration, Interpretability, Robustness Testing, and Semi-Supervised Learning

AI 服务暂时不可用，以下为来源正文，待恢复后补全翻译。

In this tutorial, we develop an end-to-end sentiment analysis workflow using the Stanford NLP IMDb Large Movie Review Dataset and compare classical machine learning with parameter-efficient transformer fine-tuning. We begin by establishing a reproducible environment and auditing the dataset for class ordering, review-length skew, duplicate leakage, and preprocessing artifacts before training a strong TF-IDF and Logistic Regression baseline. We then fine-tune DistilBERT with LoRA through PEFT, evaluate it using accuracy, macro-F1, ROC-AUC, confusion matrices, and ROC curves, and examine threshold selection and probability calibration through Expected Calibration Error and reliability analysis. Beyond headline metrics, we investigate confident errors, performance across review lengths, word-level occlusion saliency, and head-versus-tail truncation to understand how the model reaches its predictions and where long-context limitations affect performance. Finally, we use the unlabeled IMDb split for confidence-based pseudo-labeling, compare the resulting semi-supervised model against our baseline, and save the merged transformer for reusable sentiment inference. Copy CodeCopiedUse a different Browser import importlib.util, subprocess, sys, os, time, random, warnings, inspect, hashlib warnings.filterwarnings("ignore") os.environ["TOKENIZERS_PARALLELISM"] = "false" os.environ["WANDB_DISABLED"] = "true" _REQUIRED = { "transformers": "transformers", "datasets": "datasets", "peft": "peft", "accelerate": "accelerate", "sklearn": "scikit-learn", } _missing = [pkg for mod, pkg in _REQUIRED.items() if importlib.util.find_spec(mod) is None] if _missing: print(f"Installing: {', '.join(_missing)} ...") subprocess.run([sys.executable, "-m", "pip", "install", "-q", *_missing], check=True) print("Done. (If imports fail below, restart the runtime and re-run.)\n") import numpy as np import pandas as pd import torch import matplotlib.pyplot as plt from datasets import load_dataset from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.linear_model import LogisticRegression from sklearn.pipeline import make_pipeline from sklearn.metrics import (accuracy_score, f1_score, roc_auc_score, classification_report, confusion_matrix, roc_curve) from transformers import (AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer, DataCollatorWithPadding, EarlyStoppingCallback, set_seed) from peft import LoraConfig, get_peft_model, TaskType def _disable_torchao_probe(): patched = [] try: import peft.import_utils as _piu _piu.is_torchao_available = lambda: False patched.append("peft.import_utils") except Exception: pass for _name, _mod in list(sys.modules.items()): if _name.startswith("peft") and hasattr(_mod, "is_torchao_available"): _mod.is_torchao_available = lambda: False patched.append(_name) return patched try: import torchao as _tao _v = getattr(_tao, "version", "?") if tuple(int(x) for x in _v.split(".")[:2]) disabling PEFT's torchao probe: " f"{', '.join(_disable_torchao_probe())}") except Exception: _disable_torchao_probe() SEED = 42 MODEL_NAME = "distilbert-base-uncased" MAX_LEN = 256 N_TRAIN = 5000 N_EVAL = 2000 N_UNSUP = 3000 EPOCHS = 2 BATCH = 16 LR = 3e-4 FULL_RUN = False if FULL_RUN: N_TRAIN, N_EVAL, EPOCHS = 25000, 25000, 3 set_seed(SEED); random.seed(SEED); np.random.seed(SEED); torch.manual_seed(SEED) DEVICE = "cuda" if torch.cuda.is_available() else "cpu" print("=" * 79) print(f"device={DEVICE} | torch={torch.version} | " f"gpu={torch.cuda.get_device_name(0) if DEVICE=='cuda' else 'n/a'}") print("=" * 79) t0 = time.time() raw = load_dataset("stanfordnlp/imdb") print(raw, f"\nloaded in {time.time()-t0:.1f}s\n") print("--- example (truncated) ---") print("label:", raw["train"][0]["label"], "|", raw["train"][0]["text"][:300], "...\n") first_labels = np.array(raw["train"]["label"][:5]) last_labels = np.array(raw["train"]["label"][-5:]) print(f"TRAP #1 - split ordering: first 5 labels {first_labels}, " f"last 5 labels {last_labels} -> ALWAYS shuffle before subsampling.") train_full = raw["train"].shuffle(seed=SEED) test_full = raw["test"].shuffle(seed=SEED) train_ds = train_full.select(range(min(N_TRAIN, len(train_full)))) eval_ds = test_full.select(range(min(N_EVAL, len(test_full)))) print(f" after shuffle+subsample: train balance = " f"{np.bincount(train_ds['label'])}, eval balance = {np.bincount(eval_ds['label'])}") lens = np.array([len(t.split()) for t in train_full["text"]]) q = np.percentile(lens, [50, 75, 90, 95, 99]) print(f"\nTRAP #2 - length (words): median={q[0]:.0f} p75={q[1]:.0f} p90={q[2]:.0f} " f"p95={q[3]:.0f} p99={q[4]:.0f} max={lens.max()}") print(f" ~{(lens > MAX_LEN*0.75).mean()*100:.1f}% of reviews exceed MAX_LEN={MAX_LEN} " f"tokens (rough words->tokens factor 1.3). Section 9 measures what that costs.") h_tr = {hashlib.md5(t.encode()).hexdigest() for t in raw["train"]["text"]} h_te = {hashlib.md5(t.encode()).hexdigest() for t in raw["test"]["text"]} print(f"\nTRAP #3 - leakage: {len(h_tr & h_te)} exact duplicate reviews across " f"train/test; {len(raw['train'])-len(h_tr)} dupes inside train itself.") def clean(t): return t.replace(" ", " ").replace(" ", " ").strip() plt.figure(figsize=(11, 3.2)) plt.subplot(1, 2, 1) plt.hist(np.clip(lens, 0, 1000), bins=60) plt.axvline(MAX_LEN, ls="--", color="k", label=f"MAX_LEN={MAX_LEN}") plt.title("Review length (words, clipped at 1000)"); plt.legend() plt.subplot(1, 2, 2) plt.bar(["neg", "pos"], np.bincount(raw["train"]["label"])) plt.title("Train class balance (perfectly balanced)") plt.tight_layout(); plt.show() We configure the Colab environment, install the required libraries, apply the PEFT–torchao compatibility fix, and set deterministic seeds for reproducible experiments. We load the Stanford IMDb dataset, shuffle and subsample the train and test splits, and inspect class balance, review-length distributions, duplicate leakage, and HTML artifacts. We also visualize review lengths and label frequencies so we understand the dataset structure before building any models. Copy CodeCopiedUse a different Browser print("\n" + "=" * 79 + "\n3. TF-IDF BASELINE\n" + "=" * 79) Xtr = [clean(t) for t in train_ds["text"]]; ytr = np.array(train_ds["label"]) Xte = [clean(t) for t in eval_ds["text"]]; yte = np.array(eval_ds["label"]) t0 = time.time() tfidf_clf = make_pipeline( TfidfVectorizer(ngram_range=(1, 2), min_df=2, max_features=300_000, sublinear_tf=True, strip_accents="unicode"), LogisticRegression(C=8.0, max_iter=2000, n_jobs=-1), ) tfidf_clf.fit(Xtr, ytr) p_tfidf = tfidf_clf.predict_proba(Xte)[:, 1] acc_tfidf = accuracy_score(yte, p_tfidf > 0.5) auc_tfidf = roc_auc_score(yte, p_tfidf) print(f"trained in {time.time()-t0:.1f}s -> acc={acc_tfidf:.4f} auc={auc_tfidf:.4f}") vec, lr = tfidf_clf.steps[0][1], tfidf_clf.steps[1][1] feats, coefs = np.array(vec.get_feature_names_out()), lr.coef_[0] order = np.argsort(coefs) print("\nmost NEGATIVE n-grams:", ", ".join(feats[order[:12]])) print("most POSITIVE n-grams:", ", ".join(feats[order[-12:]][::-1])) print("\n" + "=" * 79 + "\n4. LoRA FINE-TUNING\n" + "=" * 79) tok = AutoTokenizer.from_pretrained(MODEL_NAME) def tokenize(batch): return tok([clean(t) for t in batch["text"]], truncation=True, max_length=MAX_LEN) tr_tok = (train_ds.map(tokenize, batched=True, remove_columns=["text"]) .rename_column("label", "labels")) ev_tok = (eval_ds.map(tokenize, batched=True, remove_columns=["text"]) .rename_column("label", "labels")) base = AutoModelForSequenceClassification.from_pretrained( MODEL_NAME, num_labels=2, id2label={0: "NEGATIVE", 1: "POSITIVE"}, label2id={"NEGATIVE": 0, "POSITIVE": 1}, ) lora_cfg = LoraConfig( task_type=TaskType.SEQ_CLS, r=16, lora_alpha=32, lora_dropout=0.05, target_modules=["q_lin", "v_lin"], modules_to_save=["pre_classifier", "classifier"], ) try: model = get_peft_model(base, lora_cfg) except ImportError as e: _disable_torchao_probe() print(f"[compat] retrying after backend probe failure: {e}") model = get_peft_model(base, lora_cfg) model.print_trainable_parameters() def compute_metrics(eval_pred): logits, labels = eval_pred probs = torch.softmax(torch.tensor(logits), dim=-1).numpy()[:, 1] preds = (probs > 0.5).astype(int) return {"accuracy": accuracy_score(labels, preds), "f1_macro": f1_score(labels, preds, average="macro"), "roc_auc": roc_auc_score(labels, probs)} _ta = inspect.signature(TrainingArguments.init).parameters _eval_key = "eval_strategy" if "eval_strategy" in _ta else "evaluation_strategy" ta_kwargs = dict( output_dir="./imdb_lora", learning_rate=LR, per_device_train_batch_size=BATCH, per_device_eval_batch_size=BATCH * 2, num_train_epochs=EPOCHS, weight_decay=0.01, warmup_ratio=0.06, logging_steps=50, save_strategy="epoch", save_total_limit=1, load_best_model_at_end=True, metric_for_best_model="accuracy", fp16=(DEVICE == "cuda"), report_to="none", seed=SEED, ) ta_kwargs[_eval_key] = "epoch" _tr = inspect.signature(Trainer.init).parameters _tok_key = "processing_class" if "processing_class" in _tr else "tokenizer" trainer = Trainer( model=model, args=TrainingArguments(ta_kwargs), train_dataset=tr_tok, eval_dataset=ev_tok, data_collator=DataCollatorWithPadding(tok), compute_metrics=compute_metrics, callbacks=[EarlyStoppingCallback(early_stopping_patience=2)], {_tok_key: tok}, ) t0 = time.time() trainer.train() print(f"\nfine-tuned in {(time.time()-t0)/60:.1f} min") We train a strong TF-IDF and Logistic Regression baseline and inspect the most influential positive and negative n-grams to establish an interpretable reference point. We then tokenize the IMDb reviews and configure DistilBERT with LoRA adapters that update only a small subset of model parameters while keeping the backbone largely frozen. We use the Hugging Face Trainer with dynamic padding, early stopping, mixed precision, and multiple evaluation metrics to fine-tune the transformer efficiently. Copy CodeCopiedUse a different Browser print("\n" + "=" * 79 + "\n5. EVALUATION\n" + "=" * 79) pred_out = trainer.predict(ev_tok) p_lora = torch.softmax(torch.tensor(pred_out.predictions), dim=-1).numpy()[:, 1] y_true = np.array(pred_out.label_ids) yhat = (p_lora > 0.5).astype(int) print(classification_report(y_true, yhat, target_names=["neg", "pos"], digits=4)) cm = confusion_matrix(y_true, yhat) fig, ax = plt.subplots(1, 2, figsize=(11, 4)) ax[0].imshow(cm, cmap="Blues") for i in range(2): for j in range(2): ax[0].text(j, i, cm[i, j], ha="center", va="center", fontsize=14) ax[0].set_xticks([0, 1], ["pred neg", "pred pos"]) ax[0].set_yticks([0, 1], ["true neg", "true pos"]); ax[0].set_title("Confusion matrix") for name, p in [("TF-IDF", p_tfidf), ("DistilBERT+LoRA", p_lora)]: fpr, tpr, _ = roc_curve(y_true, p) ax[1].plot(fpr, tpr, label=f"{name} (AUC={roc_auc_score(y_true, p):.4f})") ax[1].plot([0, 1], [0, 1], "k--", lw=0.8) ax[1].set_xlabel("FPR"); ax[1].set_ylabel("TPR"); ax[1].set_title("ROC"); ax[1].legend() plt.tight_layout(); plt.show() print("\n" + "=" * 79 + "\n6. THRESHOLD & CALIBRATION\n" + "=" * 79) ths = np.linspace(0.05, 0.95, 91) accs = [(y_true == (p_lora > t)).mean() for t in ths] best_t = ths[int(np.argmax(accs))] print(f"[email protected] = {accs[45]:.4f} | best threshold = {best_t:.2f} -> acc = {max(accs):.4f}") def expected_calibration_error(probs, labels, n_bins=10): """ECE: |confidence - accuracy| averaged over confidence bins.""" conf = np.maximum(probs, 1 - probs) correct = (probs > 0.5).astype(int) == labels bins = np.linspace(0, 1, n_bins + 1) ece, xs, ys = 0.0, [], [] for lo, hi in zip(bins[:-1], bins[1:]): m = (conf > lo) & (conf 0.5).astype(int) err["correct"] = err.pred == err.y err["confidence"] = np.maximum(err.p_pos, 1 - err.p_pos) print("--- 3 most CONFIDENT mistakes (where the model is confidently wrong) ---") for _, r in err[~err.correct].nlargest(3, "confidence").iterrows(): print(f"\n[true={'pos' if r.y else 'neg'} pred={'pos' if r.pred else 'neg'} " f"conf={r.confidence:.3f} words [truncated for AI cost control]