AI 服務暫時不可用，以下為來源正文，待恢復後補全翻譯。
In this tutorial, we build a comprehensive multimodal augmentation and robustness workflow with AugLy for images, text, and audio. We start by addressing modern dependency compatibility issues and generating deterministic synthetic datasets so the experiments remain self-contained and reproducible. We then explore AugLy’s functional and class-based APIs, metadata, and intensity tracking, probabilistic composition, bounding-box-aware transformations, and custom transforms. We extend the workflow into practical robustness experiments by benchmarking perceptual-hash copy detection under image distortions and evaluating text classifiers against adversarial perturbations, Unicode obfuscation, sanitization, and adversarial training. We also integrate audio augmentation, build a queryable metadata warehouse, and connect AugLy transformations directly to PyTorch datasets and DataLoaders, giving us an end-to-end view of augmentation as both a data-generation mechanism and a measurable robustness tool. Copy CodeCopiedUse a different Browser import subprocess, sys, importlib def _sh(cmd): print(f"$ {cmd}") subprocess.run(cmd, shell=True, check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) def _need(mod): try: importlib.import_module(mod) return False except ImportError: return True if _need("augly"): _sh("apt-get -qq install -y libmagic1 > /dev/null 2>&1") _sh(f'"{sys.executable}" -m pip install -q --no-deps augly') _sh(f'"{sys.executable}" -m pip install -q "iopath>=0.1.8" "python-magic>=0.4.22" ' f'"regex>=2021.4.4" "nlpaug==1.1.3"') import numpy as np from PIL import Image, ImageDraw, ImageFont, ImageFilter for _name, _builtin in (("float", float), ("int", int), ("bool", bool)): if not hasattr(np, _name): setattr(np, _name, _builtin) def _size(font, text): left, top, right, bottom = font.getbbox(text) return (right, bottom) if not hasattr(ImageFont.FreeTypeFont, "getsize"): ImageFont.FreeTypeFont.getsize = lambda self, t, *a, **k: _size(self, t) if not hasattr(ImageFont.FreeTypeFont, "getsize_multiline"): def _getsize_multiline(self, text, direction=None, spacing=4, features=None, language=None, stroke_width=0): lines = text.split("\n") w = max((_size(self, ln)[0] for ln in lines), default=0) h = sum(_size(self, ln)[1] for ln in lines) + spacing * (len(lines) - 1) return (w, h) ImageFont.FreeTypeFont.getsize_multiline = _getsize_multiline import os, io, json, math, random, string, textwrap, unicodedata, warnings from dataclasses import dataclass from typing import Any, Dict, List, Optional, Tuple import matplotlib.pyplot as plt import pandas as pd import augly.image as imaugs import augly.text as textaugs import augly.utils as augutils from augly.image.transforms import BaseTransform as ImageBaseTransform warnings.filterwarnings("ignore") pd.set_option("display.width", 160) SEED = 1234 random.seed(SEED) np.random.seed(SEED) print("\n" + "=" * 78) print("AugLy ready. assets at:", augutils.ASSETS_BASE_DIR) print("image augs :", len([f for f in dir(imaugs) if f[0].islower()])) print("text augs :", len([f for f in dir(textaugs) if f[0].islower()])) print("=" * 78 + "\n") def make_image(idx: int, w: int = 320, h: int = 240) -> Tuple[Image.Image, Tuple[int, int, int, int]]: """Procedurally generated 'photo' + a ground-truth bbox in pascal_voc format.""" rng = random.Random(SEED + idx) img = Image.new("RGB", (w, h), tuple(rng.randint(20, 90) for _ in range(3))) d = ImageDraw.Draw(img) for _ in range(70): x0, y0 = rng.randint(0, w), rng.randint(0, h) d.line([x0, y0, x0 + rng.randint(-60, 60), y0 + rng.randint(-60, 60)], fill=tuple(rng.randint(60, 160) for _ in range(3)), width=rng.randint(1, 3)) ow, oh = rng.randint(70, 130), rng.randint(60, 110) ox, oy = rng.randint(10, w - ow - 10), rng.randint(10, h - oh - 10) box = (ox, oy, ox + ow, oy + oh) colour = tuple(rng.randint(150, 255) for _ in range(3)) if idx % 3 == 0: d.ellipse(box, fill=colour, outline=(255, 255, 255), width=3) elif idx % 3 == 1: d.rectangle(box, fill=colour, outline=(255, 255, 255), width=3) else: d.polygon([(ox + ow // 2, oy), (ox + ow, oy + oh), (ox, oy + oh)], fill=colour, outline=(255, 255, 255)) return img, box N_IMAGES = 24 IMAGES, BOXES = zip(*[make_image(i) for i in range(N_IMAGES)]) IMAGES, BOXES = list(IMAGES), list(BOXES) DEMO_IMG, DEMO_BOX = IMAGES[0], BOXES[0] def make_text_dataset(n_per_class: int = 260): """Tiny sentiment corpus built from templates -> learnable but not trivial.""" rng = random.Random(SEED) pos_adj = ["excellent", "delightful", "superb", "charming", "brilliant", "flawless", "wonderful", "outstanding", "impressive", "lovely"] neg_adj = ["terrible", "awful", "dreadful", "disappointing", "clumsy", "broken", "miserable", "useless", "painful", "sloppy"] subj = ["the movie", "this restaurant", "the hotel room", "their support team", "the new phone", "the sequel", "this laptop", "the delivery service"] tail_p = ["and I would recommend it to anyone", "worth every rupee", "I left completely satisfied", "easily the best of the year", "it exceeded all my expectations"] tail_n = ["and I want a refund", "a total waste of money", "I left extremely frustrated", "easily the worst of the year", "it failed every expectation"] rows = [] for _ in range(n_per_class): rows.append((f"{rng.choice(subj)} was {rng.choice(pos_adj)} {rng.choice(tail_p)}", 1)) rows.append((f"{rng.choice(subj)} was {rng.choice(neg_adj)} {rng.choice(tail_n)}", 0)) rng.shuffle(rows) return [r[0] for r in rows], [r[1] for r in rows] TEXTS, LABELS = make_text_dataset() DEMO_TEXT = "The quick brown fox jumps over the lazy dog near the river bank" def make_audio(seconds: float = 2.0, sr: int = 16000) -> Tuple[np.ndarray, int]: """A chirp + harmonics + a little noise = something you can actually hear change.""" t = np.linspace(0, seconds, int(sr * seconds), endpoint=False) f = np.linspace(220, 880, t.size) sig = 0.5 * np.sin(2 * np.pi * f * t) + 0.2 * np.sin(2 * np.pi * 2 * f * t) sig += 0.02 * np.random.RandomState(SEED).randn(t.size) env = np.minimum(1.0, np.minimum(t * 8, (seconds - t) * 8)) return (sig * env).astype(np.float32), sr AUDIO, SR = make_audio() def show_grid(pairs, cols=4, title="", figsize_scale=2.9): """pairs: list of (caption, PIL.Image).""" rows = math.ceil(len(pairs) / cols) fig, axes = plt.subplots(rows, cols, figsize=(cols * figsize_scale, rows * figsize_scale)) axes = np.atleast_1d(axes).ravel() for ax, (cap, im) in zip(axes, pairs): ax.imshow(im) ax.set_title(cap, fontsize=8) ax.axis("off") for ax in axes[len(pairs):]: ax.axis("off") if title: fig.suptitle(title, fontsize=13, y=1.0) plt.tight_layout() plt.show() def as_str(out) -> str: """AugLy text augs return str for str input in some transforms, list in others.""" return out[0] if isinstance(out, list) else out print("\n### §2 IMAGE AUGMENTATION + METADATA " + "#" * 38) functional_result = imaugs.pixelization(DEMO_IMG, ratio=0.25) class_result = imaugs.Pixelization(ratio=0.25, p=1.0)(DEMO_IMG) print("functional == class:", np.array_equal(np.array(functional_result), np.array(class_result))) IMAGE_ZOO = { "blur": lambda im, m: imaugs.blur(im, radius=3.0, metadata=m), "brightness": lambda im, m: imaugs.brightness(im, factor=1.7, metadata=m), "color_jitter": lambda im, m: imaugs.color_jitter(im, brightness_factor=1.3, contrast_factor=1.4, saturation_factor=1.6, metadata=m), "crop": lambda im, m: imaugs.crop(im, x1=.15, y1=.15, x2=.85, y2=.85, metadata=m), "encoding_quality": lambda im, m: imaugs.encoding_quality(im, quality=8, metadata=m), "grayscale": lambda im, m: imaugs.grayscale(im, metadata=m), "hflip": lambda im, m: imaugs.hflip(im, metadata=m), "meme_format": lambda im, m: imaugs.meme_format(im, text="TOP TEXT", caption_height=90, metadata=m), "opacity": lambda im, m: imaugs.opacity(im, level=0.45, metadata=m), "overlay_emoji": lambda im, m: imaugs.overlay_emoji(im, opacity=0.9, emoji_size=0.35, metadata=m), "overlay_screenshot": lambda im, m: imaugs.overlay_onto_screenshot(im, metadata=m), "overlay_stripes": lambda im, m: imaugs.overlay_stripes(im, line_width=0.4, line_opacity=0.7, metadata=m), "overlay_text": lambda im, m: imaugs.overlay_text(im, opacity=0.9, metadata=m), "pad_square": lambda im, m: imaugs.pad_square(im, metadata=m), "perspective": lambda im, m: imaugs.perspective_transform(im, sigma=40.0, metadata=m), "pixelization": lambda im, m: imaugs.pixelization(im, ratio=0.15, metadata=m), "random_noise": lambda im, m: imaugs.random_noise(im, var=0.03, metadata=m), "rotate": lambda im, m: imaugs.rotate(im, degrees=17, metadata=m), "saturation": lambda im, m: imaugs.saturation(im, factor=3.0, metadata=m), "scale": lambda im, m: imaugs.scale(im, factor=0.35, metadata=m), "sharpen": lambda im, m: imaugs.sharpen(im, factor=8.0, metadata=m), "shuffle_pixels": lambda im, m: imaugs.shuffle_pixels(im, factor=0.15, metadata=m), "skew": lambda im, m: imaugs.skew(im, skew_factor=0.35, metadata=m), "vflip": lambda im, m: imaugs.vflip(im, metadata=m), } gallery, image_meta = [("ORIGINAL", DEMO_IMG)], [] for name, fn in IMAGE_ZOO.items(): m = [] try: out = fn(DEMO_IMG, m) gallery.append((f"{name}\nintensity={m[0]['intensity']:.1f}", out)) image_meta.append(m[0]) except Exception as e: print(f" [skip] {name}: {type(e).name}: {e}") show_grid(gallery, cols=5, title="§2 AugLy image augmentations (with AugLy's own intensity score)") meta_df = pd.DataFrame(image_meta)[["name", "intensity", "src_width", "src_height", "dst_width", "dst_height"]] print(meta_df.sort_values("intensity", ascending=False).head(10).to_string(index=False)) We set up AugLy in a modern Colab environment while adding compatibility shims for NumPy and Pillow. We generate deterministic synthetic image, text, and audio datasets without external downloads. We also initialize reusable visualization and utility functions before exploring image augmentation and metadata. Copy CodeCopiedUse a different Browser print("\n### §3 COMPOSITION & REPRODUCIBILITY " + "#" * 39) REUPLOAD_PIPELINE = imaugs.Compose([ imaugs.OneOf([ imaugs.OverlayOntoScreenshot(), imaugs.MemeFormat(text="LOL", caption_height=80), imaugs.OverlayStripes(line_width=0.3, line_opacity=0.5), ], p=0.9), imaugs.RandomAspectRatio(min_ratio=0.7, max_ratio=1.4, p=0.5), imaugs.RandomEmojiOverlay(p=0.7), imaugs.RandomBrightness(min_factor=0.7, max_factor=1.4, p=0.6), imaugs.EncodingQuality(quality=12, p=1.0), ]) def run_pipeline(img, seed=None): """AugLy image transforms use the global random module -> seed it for determinism.""" if seed is not None: random.seed(seed) np.random.seed(seed) meta = [] return REUPLOAD_PIPELINE(img, metadata=meta), meta a, meta_a = run_pipeline(DEMO_IMG, seed=7) b, meta_b = run_pipeline(DEMO_IMG, seed=7) c, _ = run_pipeline(DEMO_IMG, seed=99) print("same seed -> identical output:", np.array_equal(np.array(a), np.array(b))) print("applied chain (seed=7) :", " -> ".join(m["name"] for m in meta_a)) show_grid([("original", DEMO_IMG), ("seed=7", a), ("seed=7 again", b), ("seed=99", c)], cols=4, title="§3 Seeded, reproducible augmentation pipelines") print("\n### §4 BBOX-AWARE AUGMENTATION " + "#" * 45) BBOX_OPS = [ ("crop", lambda im, m, bb: imaugs.crop(im, x1=.1, y1=.1, x2=.9, y2=.9, metadata=m, bboxes=bb, bbox_format="pascal_voc")), ("hflip", lambda im, m, bb: imaugs.hflip(im, metadata=m, bboxes=bb, bbox_format="pascal_voc")), ("rotate 20", lambda im, m, bb: imaugs.rotate(im, degrees=20, metadata=m, bboxes=bb, bbox_format="pascal_voc")), ("pad", lambda im, m, bb: imaugs.pad(im, w_factor=0.25, h_factor=0.25, metadata=m, bboxes=bb, bbox_format="pascal_voc")), ("meme_format", lambda im, m, bb: imaugs.meme_format(im, text="BOXED", caption_height=80, metadata=m, bboxes=bb, bbox_format="pascal_voc")), ] def draw_box(img, box, colour=(0, 255, 0)): out = img.copy().convert("RGB") ImageDraw.Draw(out).rectangle([float(v) for v in box], outline=colour, width=4) return out bbox_panels = [("original", draw_box(DEMO_IMG, DEMO_BOX))] for label, op in BBOX_OPS: m = [] try: out = [truncated for AI cost control]