Can CLIP Tell If a Picture Has the Malfoid Look? Zero-Shot Trait Checks in Python, and the Prompt Wording That Changed the Answer

def recolour(a, region, pick, target): """Pixels in region chosen by pick() get target colour scaled by their luma.""" m = region & pick(a) if target is None or not m.any(): return a lum = (a[..., :3] @ [0.299, 0.587, 0.114]) / 255.0 ref = np.median(lum[m]) t = np.array(target, dtype=np.float64) out = a.copy() out[m] = np.clip(t * (lum[m] / ref)[:, None], 0, 255) return out 

#!/usr/bin/env python3 """Zero-shot trait checks with CLIP: does the wording of the text prompt change whether the model sees a green tie and platinum hair? pip install torch transformers pillow numpy python make_variants.py variants/ python clip_traits.py variants/ # prints the tables below Model: openai/clip-vit-base-patch32 (Hugging Face transformers). Every image is scored against every prompt once; each "prompt design" is a different way of turning those similarities into a decision. """ import json import os import sys from itertools import product import numpy as np import torch from PIL import Image, ImageOps from transformers import CLIPModel, CLIPProcessor MODEL = "openai/clip-vit-base-patch32" TIES = ["green", "red", "blue"] HAIRS = {"platinum": "platinum blonde", "golden": "golden blonde", "dark brown": "dark brown"} TEMPLATES = ["a photo of a person wearing a {} tie.", "a drawing of a person wearing a {} tie.", "an illustration of someone with a {} striped tie.", "a painting of a person in a {} necktie.", "a cartoon of a student wearing a {} school tie."] HAIR_TEMPLATES = ["a photo of a person with {} hair.", "a drawing of a woman with long {} hair.", "an illustration of someone with {} hair.", "a painting of a person with {} hair.", "a cartoon of a student with long {} hair."] PLATINUM_WORDS = ["platinum blonde", "white-blonde", "silver", "very pale blonde", "white", "blonde"] def load(d): rows = [json.loads(l) for l in open(os.path.join(d, "labels.jsonl"))] return rows, [Image.open(os.path.join(d, r["file_name"])).convert("RGB") for r in rows] def warm_cast(img): """A strong warm white-balance error, like a selfie under candles or tungsten.""" a = np.asarray(img, dtype=np.float64) * [1.12, 0.97, 0.72] return Image.fromarray(np.clip(a, 0, 255).astype(np.uint8)) def pad_square(img, fill=(0, 0, 0)): w, h = img.size s = max(w, h) return ImageOps.pad(img, (s, s), color=fill) class Clip: def __init__(self): self.m = CLIPModel.from_pretrained(MODEL).eval() self.p = CLIPProcessor.from_pretrained(MODEL) @torch.no_grad() def images(self, imgs): x = self.p(images=imgs, return_tensors="pt") e = self.m.get_image_features(**x) e = e if isinstance(e, torch.Tensor) else e.pooler_output return torch.nn.functional.normalize(e, dim=-1).numpy() @torch.no_grad() def texts(self, ts): x = self.p(text=ts, return_tensors="pt", padding=True) e = self.m.get_text_features(**x) e = e if isinstance(e, torch.Tensor) else e.pooler_output return torch.nn.functional.normalize(e, dim=-1).numpy() def classes(self, prompts_per_class): """One unit vector per class: mean of its prompts' embeddings (CLIP's ensembling).""" out = [] for ps in prompts_per_class: e = self.texts(ps).mean(axis=0) out.append(e / np.linalg.norm(e)) return np.stack(out) def acc(pred, truth): return sum(p == t for p, t in zip(pred, truth)) / len(truth) def balanced(pred, truth): """Mean of per-class recall, so 8 positives vs 64 negatives can't hide.""" cls = sorted(set(truth)) return sum(sum(p == t == c for p, t in zip(pred, truth)) / truth.count(c) for c in cls) / len(cls) def main(d): rows, imgs = load(d) clip = Clip() views = {"centre crop (default)": clip.images(imgs), "pad to square": clip.images([pad_square(i) for i in imgs])} tie_t = [r["tie"] for r in rows] hair_t = [r["hair"] for r in rows] look_t = [r["malfoid_look"] for r in rows] designs = { "tie": { "one word": [[f"{c} tie"] for c in TIES], "one sentence": [[f"a portrait of a person wearing a {c} striped tie"] for c in TIES], "5-template ensemble": [[t.format(c) for t in TEMPLATES] for c in TIES], }, "hair": { "one word": [[f"{w} hair"] for w in HAIRS.values()], "one sentence": [[f"a portrait of a woman with long {w} hair"] for w in HAIRS.values()], "5-template ensemble": [[t.format(w) for t in HAIR_TEMPLATES] for w in HAIRS.values()], }, } print(f"{len(rows)} images: 8 synthetic sitters x 3 tie colours x 3 hair colours") for view, E in views.items(): print(f"\n== image preprocessing: {view}") print(f"{'trait':<5} {'prompt design':<22} {'3-way accuracy':>14}") for trait, ds in designs.items(): truth = tie_t if trait == "tie" else hair_t names = TIES if trait == "tie" else list(HAIRS) for name, prompts in ds.items(): pred = [names[i] for i in (E @ clip.classes(prompts).T).argmax(1)] print(f"{trait:<5} {name:<22} {acc(pred, truth):>13.0%}") E = views["pad to square"] print("\n== binary 'is the tie green?' (pad to square): what the other prompt says") green_t = [t == "green" for t in tie_t] for label, neg in [("'a tie that is not green'", ["a person wearing a tie that is not green"]), ("'no green tie'", ["a person with no green tie"]), ("'a red or blue tie'", ["a person wearing a red or blue tie"]), ("red + blue as two classes", None)]: pos = ["a person wearing a green tie"] if neg is None: C = clip.classes([pos, ["a person wearing a red tie"], ["a person wearing a blue tie"]]) pred = list((E @ C.T).argmax(1) == 0) else: C = clip.classes([pos, neg]) pred = list((E @ C.T).argmax(1) == 0) leaked = {c: sum(p and t == c for p, t in zip(pred, tie_t)) for c in ("red", "blue")} print(f" negative = {label:<28} balanced accuracy {balanced(pred, green_t):.0%} " f"(green found {sum(p and t for p, t in zip(pred, green_t))}/24, passed as green: red {leaked['red']}/24, blue {leaked['blue']}/24)") print("\n== which word for 'platinum'? 3-way hair accuracy (pad to square, 5-template ensemble)") for w in PLATINUM_WORDS: P = [[t.format(w) for t in HAIR_TEMPLATES]] + [[t.format(x) for t in HAIR_TEMPLATES] for x in ("golden blonde", "dark brown")] pred = [list(HAIRS)[i] for i in (E @ clip.classes(P).T).argmax(1)] rec = sum(p == t == "platinum" for p, t in zip(pred, hair_t)) / 24 print(f" {w!r:<20} accuracy {acc(pred, hair_t):.0%} platinum recall {rec:.0%}") print("\n== warm candle-light cast (R x1.12, G x0.97, B x0.72), pad to square, 5-template ensembles") warm = clip.images([pad_square(warm_cast(i)) for i in imgs]) for trait, truth, names in (("tie", tie_t, TIES), ("hair", hair_t, list(HAIRS))): pred = [names[i] for i in (warm @ clip.classes(designs[trait]["5-template ensemble"]).T).argmax(1)] conf = {t: {p: 0 for p in names} for t in names} for p, t in zip(pred, truth): conf[t][p] += 1 print(f" {trait:<5} accuracy {acc(pred, truth):.0%} " + "; ".join( f"{t} -> " + ", ".join(f"{p} {n}" for p, n in conf[t].items() if n) for t in names)) print("\n== 'has the Malfoid look' (green tie AND platinum hair): 8 of 72 images") combos = list(product(TIES, HAIRS)) one = clip.classes([[f"a portrait of a woman with long {HAIRS[h]} hair wearing a {t} striped tie"] for t, h in combos]) pred_one = [combos[i] == ("green", "platinum") for i in (E @ one.T).argmax(1)] tie_c = clip.classes(designs["tie"]["5-template ensemble"]) hair_c = clip.classes(designs["hair"]["5-template ensemble"]) pred_two = [TIES[a] == "green" and list(HAIRS)[b] == "platinum" for a, b in zip((E @ tie_c.T).argmax(1), (E @ hair_c.T).argmax(1))] for label, pred in [("one 9-way prompt describing both traits", pred_one), ("two separate trait checks, AND-ed", pred_two)]: tp = sum(p and t for p, t in zip(pred, look_t)) fp = sum(p and not t for p, t in zip(pred, look_t)) print(f" {label:<42} balanced accuracy {balanced(pred, look_t):.0%} (look found {tp}/8, false 'look' {fp}/64)") if __name__ == "__main__": main(...