471 lines
19 KiB
Python
471 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Compare an approved design master with a normal-render Runtime capture.
|
|
|
|
Static artwork is compared with dynamic search regions masked out. Dynamic
|
|
geometry is measured from rendered pixels by subtracting the same clean static
|
|
master from both the approved design and Runtime capture. Declared Runtime
|
|
bounds are never accepted as evidence of rendered geometry.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from collections import deque
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from PIL import Image, ImageChops, ImageDraw, ImageFilter
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--approved", type=Path, required=True)
|
|
parser.add_argument("--runtime", type=Path, required=True)
|
|
parser.add_argument("--contract", type=Path, required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
parser.add_argument("--allow-fail", action="store_true")
|
|
return parser.parse_args()
|
|
|
|
|
|
def rect(bounds: list[float], size: tuple[int, int]) -> tuple[int, int, int, int]:
|
|
width, height = size
|
|
x, y, w, h = bounds
|
|
return (
|
|
max(0, min(width, round(x * width))),
|
|
max(0, min(height, round(y * height))),
|
|
max(0, min(width, round((x + w) * width))),
|
|
max(0, min(height, round((y + h) * height))),
|
|
)
|
|
|
|
|
|
def normalized_bounds(
|
|
box: tuple[int, int, int, int], size: tuple[int, int]
|
|
) -> list[float]:
|
|
left, top, right, bottom = box
|
|
width, height = size
|
|
return [
|
|
left / width,
|
|
top / height,
|
|
(right - left) / width,
|
|
(bottom - top) / height,
|
|
]
|
|
|
|
|
|
def mean_error(left: Image.Image, right: Image.Image, mask: Image.Image | None = None) -> float:
|
|
diff = ImageChops.difference(left.convert("RGB"), right.convert("RGB"))
|
|
if mask is None:
|
|
histogram = diff.histogram()
|
|
total = sum(value * (index % 256) for index, value in enumerate(histogram))
|
|
pixels = left.width * left.height * 3
|
|
return total / max(1, pixels * 255)
|
|
|
|
mask = mask.convert("L")
|
|
included = sum(1 for value in mask.getdata() if value)
|
|
if included == 0:
|
|
return 0.0
|
|
total = 0
|
|
for channel in diff.split():
|
|
histogram = channel.histogram(mask=mask)
|
|
total += sum(value * count for value, count in enumerate(histogram))
|
|
return total / (included * 3 * 255)
|
|
|
|
|
|
def contained_canvas(image: Image.Image, aspect: float, policy: Any) -> Image.Image:
|
|
if isinstance(policy, list):
|
|
return image.crop(rect(policy, image.size))
|
|
width, height = image.size
|
|
current = width / height
|
|
if current >= aspect:
|
|
canvas_width = round(height * aspect)
|
|
left = (width - canvas_width) // 2
|
|
return image.crop((left, 0, left + canvas_width, height))
|
|
canvas_height = round(width / aspect)
|
|
top = (height - canvas_height) // 2
|
|
return image.crop((0, top, width, top + canvas_height))
|
|
|
|
|
|
def resolve_contract_path(contract_path: Path, value: str) -> Path:
|
|
path = Path(value)
|
|
if not path.is_absolute():
|
|
path = contract_path.parent / path
|
|
return path.resolve()
|
|
|
|
|
|
def raw_difference_mask(left: Image.Image, right: Image.Image, threshold: float) -> Image.Image:
|
|
diff = ImageChops.difference(left.convert("RGB"), right.convert("RGB"))
|
|
red, green, blue = diff.split()
|
|
maximum = ImageChops.lighter(ImageChops.lighter(red, green), blue)
|
|
cutoff = round(threshold * 255) if threshold <= 1.0 else round(threshold)
|
|
return maximum.point(lambda value: 255 if value >= cutoff else 0, mode="L")
|
|
|
|
|
|
def remove_small_components(mask: Image.Image, minimum_area: int) -> Image.Image:
|
|
if minimum_area <= 1:
|
|
return mask
|
|
width, height = mask.size
|
|
pixels = mask.load()
|
|
visited = bytearray(width * height)
|
|
kept = Image.new("L", mask.size, 0)
|
|
kept_pixels = kept.load()
|
|
for y in range(height):
|
|
for x in range(width):
|
|
offset = y * width + x
|
|
if visited[offset] or not pixels[x, y]:
|
|
continue
|
|
visited[offset] = 1
|
|
pending: deque[tuple[int, int]] = deque([(x, y)])
|
|
component: list[tuple[int, int]] = []
|
|
while pending:
|
|
current_x, current_y = pending.popleft()
|
|
component.append((current_x, current_y))
|
|
for next_x, next_y in (
|
|
(current_x - 1, current_y),
|
|
(current_x + 1, current_y),
|
|
(current_x, current_y - 1),
|
|
(current_x, current_y + 1),
|
|
):
|
|
if next_x < 0 or next_y < 0 or next_x >= width or next_y >= height:
|
|
continue
|
|
next_offset = next_y * width + next_x
|
|
if visited[next_offset] or not pixels[next_x, next_y]:
|
|
continue
|
|
visited[next_offset] = 1
|
|
pending.append((next_x, next_y))
|
|
if len(component) >= minimum_area:
|
|
for kept_x, kept_y in component:
|
|
kept_pixels[kept_x, kept_y] = 255
|
|
return kept
|
|
|
|
|
|
def prepare_detection_mask(
|
|
image: Image.Image,
|
|
static_master: Image.Image,
|
|
search_bounds: list[float],
|
|
detection: dict[str, Any],
|
|
) -> Image.Image:
|
|
threshold = float(detection.get("difference_threshold", 0.10))
|
|
mask = raw_difference_mask(image, static_master, threshold)
|
|
outside = Image.new("L", mask.size, 0)
|
|
search_box = rect(search_bounds, mask.size)
|
|
outside.paste(mask.crop(search_box), search_box[:2])
|
|
mask = outside
|
|
draw = ImageDraw.Draw(mask)
|
|
for excluded in detection.get("exclude_bounds", []):
|
|
draw.rectangle(rect(excluded, mask.size), fill=0)
|
|
median_size = int(detection.get("median_filter_px", 0))
|
|
if median_size > 1:
|
|
if median_size % 2 == 0:
|
|
median_size += 1
|
|
mask = mask.filter(ImageFilter.MedianFilter(median_size))
|
|
opening_size = int(detection.get("morphology_open_px", 0))
|
|
if opening_size > 1:
|
|
if opening_size % 2 == 0:
|
|
opening_size += 1
|
|
mask = mask.filter(ImageFilter.MinFilter(opening_size))
|
|
mask = mask.filter(ImageFilter.MaxFilter(opening_size))
|
|
return remove_small_components(mask, int(detection.get("minimum_component_area_px", 1)))
|
|
|
|
|
|
def measure_mask(mask: Image.Image, search_bounds: list[float]) -> dict[str, Any]:
|
|
box = mask.getbbox()
|
|
foreground_pixels = sum(1 for value in mask.getdata() if value)
|
|
search_box = rect(search_bounds, mask.size)
|
|
search_area = max(1, (search_box[2] - search_box[0]) * (search_box[3] - search_box[1]))
|
|
if box is None:
|
|
return {
|
|
"bounds": None,
|
|
"foreground_pixels": 0,
|
|
"search_occupancy": 0.0,
|
|
"bounds_occupancy": 0.0,
|
|
}
|
|
bounds_area = max(1, (box[2] - box[0]) * (box[3] - box[1]))
|
|
return {
|
|
"bounds": normalized_bounds(box, mask.size),
|
|
"foreground_pixels": foreground_pixels,
|
|
"search_occupancy": foreground_pixels / search_area,
|
|
"bounds_occupancy": foreground_pixels / bounds_area,
|
|
}
|
|
|
|
|
|
def maximum_delta(left: list[float], right: list[float], indexes: tuple[int, ...]) -> float:
|
|
return max(abs(left[index] - right[index]) for index in indexes)
|
|
|
|
|
|
def offset_box_x(box: tuple[int, int, int, int], offset: int) -> tuple[int, int, int, int]:
|
|
return (box[0] + offset, box[1], box[2] + offset, box[3])
|
|
|
|
|
|
def dynamic_check(
|
|
region: dict[str, Any],
|
|
approved: Image.Image,
|
|
runtime: Image.Image,
|
|
static_master: Image.Image,
|
|
defaults: dict[str, Any],
|
|
) -> tuple[dict[str, Any], Image.Image, Image.Image]:
|
|
detection = dict(defaults.get("dynamic_detection", {}))
|
|
detection.update(region.get("detection", {}))
|
|
search_bounds = region.get("search_bounds")
|
|
if not isinstance(search_bounds, list) or len(search_bounds) != 4:
|
|
raise ValueError(f"dynamic region {region['id']} requires search_bounds")
|
|
approved_mask = prepare_detection_mask(approved, static_master, search_bounds, detection)
|
|
runtime_mask = prepare_detection_mask(runtime, static_master, search_bounds, detection)
|
|
approved_measurement = measure_mask(approved_mask, search_bounds)
|
|
runtime_measurement = measure_mask(runtime_mask, search_bounds)
|
|
result: dict[str, Any] = {
|
|
"id": region["id"],
|
|
"mode": region["mode"],
|
|
"measurement_source": "rendered_pixel_delta_from_clean_static_master",
|
|
"search_bounds": search_bounds,
|
|
"approved_measurement": approved_measurement,
|
|
"runtime_measurement": runtime_measurement,
|
|
}
|
|
if "runtime_bounds" in region:
|
|
result["declared_runtime_bounds_ignored"] = region["runtime_bounds"]
|
|
|
|
approved_bounds = approved_measurement["bounds"]
|
|
runtime_bounds = runtime_measurement["bounds"]
|
|
minimum_pixels = int(detection.get("minimum_foreground_pixels", 1))
|
|
minimum_occupancy = float(detection.get("minimum_search_occupancy", 0.0))
|
|
presence_pass = (
|
|
runtime_bounds is not None
|
|
and runtime_measurement["foreground_pixels"] >= minimum_pixels
|
|
and runtime_measurement["search_occupancy"] >= minimum_occupancy
|
|
)
|
|
result["presence"] = {
|
|
"minimum_foreground_pixels": minimum_pixels,
|
|
"minimum_search_occupancy": minimum_occupancy,
|
|
"pass": presence_pass,
|
|
}
|
|
|
|
comparison = region.get("comparison", "approved_delta")
|
|
if comparison == "runtime_presence":
|
|
anchor_delta = 0.0
|
|
size_delta = 0.0
|
|
occupancy_delta = 0.0
|
|
elif comparison == "runtime_target":
|
|
target_bounds = region.get("target_bounds")
|
|
if not isinstance(target_bounds, list) or len(target_bounds) != 4:
|
|
raise ValueError(f"runtime-target region {region['id']} requires target_bounds")
|
|
anchor_delta = maximum_delta(runtime_bounds, target_bounds, (0, 1)) if runtime_bounds else 1.0
|
|
size_delta = maximum_delta(runtime_bounds, target_bounds, (2, 3)) if runtime_bounds else 1.0
|
|
occupancy_delta = 0.0
|
|
result["target_bounds"] = target_bounds
|
|
else:
|
|
if approved_bounds is None or runtime_bounds is None:
|
|
anchor_delta = 1.0
|
|
size_delta = 1.0
|
|
occupancy_delta = 1.0
|
|
else:
|
|
anchor_delta = maximum_delta(approved_bounds, runtime_bounds, (0, 1))
|
|
size_delta = maximum_delta(approved_bounds, runtime_bounds, (2, 3))
|
|
occupancy_delta = abs(
|
|
approved_measurement["bounds_occupancy"]
|
|
- runtime_measurement["bounds_occupancy"]
|
|
)
|
|
anchor_tolerance = float(region.get("anchor_tolerance", defaults.get("anchor_delta", 0.03)))
|
|
size_tolerance = float(region.get("size_tolerance", defaults.get("focal_scale_delta", 0.08)))
|
|
occupancy_tolerance = float(
|
|
region.get("occupancy_tolerance", defaults.get("occupancy_delta", 0.20))
|
|
)
|
|
geometry_pass = anchor_delta <= anchor_tolerance and size_delta <= size_tolerance
|
|
occupancy_pass = comparison in ("runtime_presence", "runtime_target") or occupancy_delta <= occupancy_tolerance
|
|
result.update(
|
|
{
|
|
"comparison": comparison,
|
|
"anchor_delta": anchor_delta,
|
|
"anchor_tolerance": anchor_tolerance,
|
|
"size_delta": size_delta,
|
|
"size_tolerance": size_tolerance,
|
|
"occupancy_delta": occupancy_delta,
|
|
"occupancy_tolerance": occupancy_tolerance,
|
|
"pass": presence_pass and geometry_pass and occupancy_pass,
|
|
}
|
|
)
|
|
return result, approved_mask, runtime_mask
|
|
|
|
|
|
def dynamic_exclusion_mask(regions: list[dict[str, Any]], size: tuple[int, int]) -> Image.Image:
|
|
mask = Image.new("L", size, 255)
|
|
draw = ImageDraw.Draw(mask)
|
|
for region in regions:
|
|
if region["mode"] == "static_visual":
|
|
continue
|
|
bounds = region.get("static_exclusion_bounds", region.get("search_bounds"))
|
|
if isinstance(bounds, list) and len(bounds) == 4:
|
|
draw.rectangle(rect(bounds, size), fill=0)
|
|
return mask
|
|
|
|
|
|
def run_verification(
|
|
approved: Image.Image,
|
|
runtime_original: Image.Image,
|
|
static_master: Image.Image,
|
|
contract: dict[str, Any],
|
|
output: Path,
|
|
approved_path: Path,
|
|
runtime_path: Path,
|
|
static_master_path: Path,
|
|
) -> dict[str, Any]:
|
|
approved_meta = contract["approved_master"]
|
|
runtime_meta = contract["runtime_canvas"]
|
|
defaults = contract.get("defaults", {})
|
|
approved_aspect = float(approved_meta["aspect"])
|
|
aspect_tolerance = float(defaults.get("aspect_ratio_delta", 0.01))
|
|
static_error_tolerance = float(defaults.get("static_region_mean_error", 0.12))
|
|
analysis_width = int(defaults.get("analysis_width", 768))
|
|
|
|
if approved.size != static_master.size:
|
|
raise ValueError("approved design and clean static master must have identical dimensions")
|
|
runtime_canvas = contained_canvas(
|
|
runtime_original, approved_aspect, runtime_meta.get("bounds", "auto_centered")
|
|
)
|
|
runtime_aspect = runtime_canvas.width / runtime_canvas.height
|
|
aspect_delta = abs(runtime_aspect - approved_aspect) / approved_aspect
|
|
analysis_size = (analysis_width, round(analysis_width / approved_aspect))
|
|
approved_analysis = approved.resize(analysis_size, Image.Resampling.LANCZOS)
|
|
runtime_analysis = runtime_canvas.resize(analysis_size, Image.Resampling.LANCZOS)
|
|
static_analysis = static_master.resize(analysis_size, Image.Resampling.LANCZOS)
|
|
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
side = Image.new("RGB", (analysis_size[0] * 2, analysis_size[1]), "#17252c")
|
|
side.paste(approved_analysis, (0, 0))
|
|
side.paste(runtime_analysis, (analysis_size[0], 0))
|
|
side.save(output / "side-by-side.png")
|
|
diff = ImageChops.difference(approved_analysis, runtime_analysis)
|
|
diff.point(lambda value: min(255, value * 3)).save(output / "pixel-diff.png")
|
|
Image.blend(approved_analysis, runtime_analysis, 0.5).save(output / "alpha-overlay.png")
|
|
|
|
regions: list[dict[str, Any]] = contract.get("regions", [])
|
|
checks: list[dict[str, Any]] = [
|
|
{
|
|
"id": "aspect_ratio",
|
|
"expected": approved_aspect,
|
|
"actual": runtime_aspect,
|
|
"relative_delta": aspect_delta,
|
|
"threshold": aspect_tolerance,
|
|
"pass": aspect_delta <= aspect_tolerance,
|
|
}
|
|
]
|
|
measured_bounds: dict[str, tuple[list[float] | None, list[float] | None]] = {}
|
|
dynamic_masks: list[Image.Image] = []
|
|
static_mask = dynamic_exclusion_mask(regions, analysis_size)
|
|
for region in regions:
|
|
if region["mode"] == "static_visual":
|
|
region_box = rect(region["approved_bounds"], analysis_size)
|
|
approved_crop = static_analysis.crop(region_box)
|
|
runtime_crop = runtime_analysis.crop(region_box)
|
|
included = static_mask.crop(region_box)
|
|
error = mean_error(approved_crop, runtime_crop, included)
|
|
checks.append(
|
|
{
|
|
"id": region["id"],
|
|
"mode": region["mode"],
|
|
"comparison_source": "clean_static_master_with_dynamic_search_regions_masked",
|
|
"approved_bounds": region["approved_bounds"],
|
|
"mean_error": error,
|
|
"threshold": float(region.get("mean_error_tolerance", static_error_tolerance)),
|
|
"pass": error
|
|
<= float(region.get("mean_error_tolerance", static_error_tolerance)),
|
|
}
|
|
)
|
|
continue
|
|
result, approved_mask, runtime_mask = dynamic_check(
|
|
region, approved_analysis, runtime_analysis, static_analysis, defaults
|
|
)
|
|
checks.append(result)
|
|
measured_bounds[region["id"]] = (
|
|
result["approved_measurement"]["bounds"],
|
|
result["runtime_measurement"]["bounds"],
|
|
)
|
|
dynamic_masks.append(runtime_mask)
|
|
|
|
combined_dynamic_mask = Image.new("L", analysis_size, 0)
|
|
for mask in dynamic_masks:
|
|
combined_dynamic_mask = ImageChops.lighter(combined_dynamic_mask, mask)
|
|
combined_dynamic_mask.save(output / "runtime-dynamic-mask.png")
|
|
static_mask.save(output / "static-comparison-mask.png")
|
|
|
|
annotated = side.copy()
|
|
draw = ImageDraw.Draw(annotated)
|
|
for region in regions:
|
|
if region["mode"] == "static_visual":
|
|
approved_box = rect(region["approved_bounds"], analysis_size)
|
|
draw.rectangle(approved_box, outline="#f3d991", width=3)
|
|
draw.rectangle(offset_box_x(approved_box, analysis_size[0]), outline="#d97964", width=3)
|
|
continue
|
|
approved_bounds, runtime_bounds = measured_bounds[region["id"]]
|
|
if approved_bounds is not None:
|
|
draw.rectangle(rect(approved_bounds, analysis_size), outline="#f3d991", width=3)
|
|
if runtime_bounds is not None:
|
|
runtime_box = rect(runtime_bounds, analysis_size)
|
|
draw.rectangle(offset_box_x(runtime_box, analysis_size[0]), outline="#d97964", width=3)
|
|
annotated.save(output / "annotated-bounds.png")
|
|
|
|
passed = all(bool(check["pass"]) for check in checks)
|
|
return {
|
|
"schema_version": 2,
|
|
"approved": str(approved_path),
|
|
"runtime": str(runtime_path),
|
|
"clean_static_master": str(static_master_path),
|
|
"approved_size": list(approved.size),
|
|
"runtime_size": list(runtime_original.size),
|
|
"runtime_canvas_size": list(runtime_canvas.size),
|
|
"analysis_size": list(analysis_size),
|
|
"checks": checks,
|
|
"result": "PASS" if passed else "FAIL_VISUAL_FIDELITY",
|
|
"artifacts": [
|
|
"side-by-side.png",
|
|
"pixel-diff.png",
|
|
"alpha-overlay.png",
|
|
"annotated-bounds.png",
|
|
"runtime-dynamic-mask.png",
|
|
"static-comparison-mask.png",
|
|
],
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
contract: dict[str, Any] = json.loads(args.contract.read_text(encoding="utf-8"))
|
|
static_meta = contract.get("clean_static_master")
|
|
if not isinstance(static_meta, dict) or not static_meta.get("path"):
|
|
raise SystemExit("visual-fidelity contract requires clean_static_master.path")
|
|
static_master_path = resolve_contract_path(args.contract, str(static_meta["path"]))
|
|
static_master_sha256 = hashlib.sha256(static_master_path.read_bytes()).hexdigest()
|
|
expected_static_sha256 = str(static_meta.get("sha256", ""))
|
|
report = run_verification(
|
|
Image.open(args.approved).convert("RGB"),
|
|
Image.open(args.runtime).convert("RGB"),
|
|
Image.open(static_master_path).convert("RGB"),
|
|
contract,
|
|
args.output,
|
|
args.approved,
|
|
args.runtime,
|
|
static_master_path,
|
|
)
|
|
report["clean_static_master"] = str(static_meta["path"])
|
|
static_hash_pass = bool(expected_static_sha256) and static_master_sha256 == expected_static_sha256
|
|
report["clean_static_master_sha256"] = static_master_sha256
|
|
report["checks"].insert(
|
|
0,
|
|
{
|
|
"id": "clean_static_master_hash",
|
|
"expected": expected_static_sha256,
|
|
"actual": static_master_sha256,
|
|
"pass": static_hash_pass,
|
|
},
|
|
)
|
|
if not static_hash_pass:
|
|
report["result"] = "FAIL_VISUAL_FIDELITY"
|
|
(args.output / "report.json").write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
|
)
|
|
print(json.dumps(report, ensure_ascii=False))
|
|
return 0 if report["result"] == "PASS" or args.allow_fail else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|