From e51c1116e104a920c434658adc8d4dc5279981a1 Mon Sep 17 00:00:00 2001 From: Johnny Zhang Date: Fri, 21 Aug 2026 14:00:39 +0800 Subject: [PATCH 1/2] =?UTF-8?q?fix(video):=20=E9=A6=96=E5=B8=A7=E6=8C=89?= =?UTF-8?q?=E7=9B=AE=E6=A0=87=E7=94=BB=E5=B8=83=E7=AD=89=E6=AF=94=E6=94=BE?= =?UTF-8?q?=E5=A4=A7=E5=B9=B6=E6=98=BE=E5=BC=8F=E5=90=88=E6=88=90=E5=BA=95?= =?UTF-8?q?=E8=89=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RGBA 首帧不再 convert("RGB") 读未定义的透明像素 RGB，改为合成到具名底色； thumbnail 只缩不放换成双向等比 contain，小 sprite 先放大再补边。 Closes #509 --- .../src/windup_framework/providers/sufy.py | 31 ++++- backend/tests/test_sufy_video_download.py | 125 ++++++++++++++++++ 2 files changed, 149 insertions(+), 7 deletions(-) diff --git a/backend/packages/framework/src/windup_framework/providers/sufy.py b/backend/packages/framework/src/windup_framework/providers/sufy.py index bf1d463b..da6c338f 100644 --- a/backend/packages/framework/src/windup_framework/providers/sufy.py +++ b/backend/packages/framework/src/windup_framework/providers/sufy.py @@ -45,24 +45,41 @@ DEFAULT_VIDEO_MODEL = "kling-v2-5-turbo" -def _fit_first_frame(frame: bytes, size: str) -> bytes: - """首帧 bytes → 等比缩放 + 背景色补边到目标尺寸 → JPG(RGB,q90) bytes。 +#: 透明首帧合成到不透明视频输入时的底色。中灰而不是黑:抠图靠主体与底色的距离 +#: 判前景,黑底会把角色的暗部判成背景(#497 的方向已实测为"被抠掉的是最暗部"), +#: 白底对浅色角色同理。中灰对两端都不偏。 +_FIRST_FRAME_BG = (128, 128, 128) + + +def _fit_first_frame(frame: bytes, size: str, *, background: tuple[int, int, int] = _FIRST_FRAME_BG) -> bytes: + """首帧 bytes → 等比缩放(可放大) + 补边到目标尺寸 → JPG(RGB,q90) bytes。 不强拉到目标尺寸(母版多为横幅,强压成方会把角色压成瘦长鬼影);JPG 因 PNG base64 会 VENDOR_FAILED(实测)。 这一步同时是 kling 系"输出画幅"的唯一控制点:kling 的 i2v 端点没有 resolution/size 字段,成片画幅跟随首帧,所以 ``size`` 只能在这里生效。 + + 小于目标画布的输入必须**放大**:128x128 的 sprite 原尺寸贴进 1280x720 只占 13% 高, + 等于自愿把主体有效分辨率砍掉七分之六,之后无论 i2v 还是重抠图都补不回来。 """ from PIL import Image w, h = (int(x) for x in size.split("x")) - im = Image.open(io.BytesIO(frame)).convert("RGB") - pad = im.getpixel((0, 0)) - fitted = im.copy() - fitted.thumbnail((w, h), Image.LANCZOS) + im = Image.open(io.BytesIO(frame)) + if im.mode in ("RGBA", "LA") or (im.mode == "P" and "transparency" in im.info): + im = im.convert("RGBA") + flat = Image.new("RGB", im.size, background) + flat.paste(im, (0, 0), im) # 不能 convert("RGB"):透明像素的 RGB 未定义 + im, pad = flat, background + else: + im = im.convert("RGB") + pad = im.getpixel((0, 0)) # 不透明输入沿用角点色,补边与画面自身背景连成一片 + scale = min(w/im.width, h/im.height) + tw, th = max(1, round(im.width*scale)), max(1, round(im.height*scale)) + fitted = im.resize((tw, th), Image.LANCZOS) canvas = Image.new("RGB", (w, h), pad) - canvas.paste(fitted, ((w - fitted.width) // 2, (h - fitted.height) // 2)) + canvas.paste(fitted, ((w - tw)//2, (h - th)//2)) buf = io.BytesIO() canvas.save(buf, "JPEG", quality=90) return buf.getvalue() diff --git a/backend/tests/test_sufy_video_download.py b/backend/tests/test_sufy_video_download.py index 2b9da701..69c90a1d 100644 --- a/backend/tests/test_sufy_video_download.py +++ b/backend/tests/test_sufy_video_download.py @@ -888,3 +888,128 @@ def test_non_positive_first_poll_is_rejected_at_construction(): for bad in (0, -1.0): with pytest.raises(ValueError, match="first_poll_after"): SufyVideoProvider(config=cfg, first_poll_after=bad) + + +def _sprite(w: int, h: int, *, alpha: bool, subject_ratio: float = 0.5, + void_rgb: tuple[int, int, int] = (0, 0, 0)) -> bytes: + """一张 sprite：正中一块暗色主体，四周是背景。 + + ``alpha=True`` 时四周真透明，且透明像素的 RGB 填 ``void_rgb`` —— 透明像素的 RGB 是 + 未定义值，测试必须能证明结果不随它变。 + """ + import io as _io + + from PIL import Image as _Image + + sh = max(1, round(h * subject_ratio)) + sw = max(1, round(w * 0.4)) + if alpha: + im = _Image.new("RGBA", (w, h), (*void_rgb, 0)) + im.paste((10, 10, 10, 255), ((w - sw) // 2, (h - sh) // 2, (w + sw) // 2, (h + sh) // 2)) + else: + im = _Image.new("RGB", (w, h), (200, 200, 200)) + im.paste((10, 10, 10), ((w - sw) // 2, (h - sh) // 2, (w + sw) // 2, (h + sh) // 2)) + buf = _io.BytesIO() + im.save(buf, "PNG") + return buf.getvalue() + + +def _submitted_first_frame(frame: bytes, size: str = "1280x720"): + import base64 as _b64 + import io as _io + + from PIL import Image as _Image + + seen: dict = {} + _video_provider(_i2v_handler(seen)).i2v(frame, "x", size=size) + uri = seen["body"]["input_reference"] + return _Image.open(_io.BytesIO(_b64.b64decode(uri.split(",", 1)[1]))) + + +def _subject_height(im) -> int: + """暗色主体在成品里的高度（像素）。JPEG 会糊边，阈值取宽一点。""" + import numpy as _np + + a = _np.asarray(im.convert("L")) + ys, _ = _np.where(a < 90) + return int(ys.max() - ys.min() + 1) if ys.size else 0 + + +@pytest.mark.parametrize("src_w,src_h", [(64, 64), (128, 128), (256, 256), (200, 300), (400, 200)]) +@pytest.mark.parametrize("alpha", [True, False]) +def test_small_first_frame_is_enlarged_instead_of_pasted_at_source_size(src_w, src_h, alpha): + """小于目标画布的输入必须等比放大后再补边，不能原尺寸贴进去。 + + 只缩不放会把主体的有效分辨率按 (源边长/画布边长) 砍掉:128x128 贴进 1280x720 只占 + 13% 高,之后 i2v 与重抠图都补不回来（#509）。判据取"主体占内容区高度的比例"而不是 + 绝对像素:它对画布尺寸与长宽比都成立,且正是用户看到的那个量。 + """ + W, H, ratio = 1280, 720, 0.5 + im = _submitted_first_frame(_sprite(src_w, src_h, alpha=alpha, subject_ratio=ratio)) + assert im.size == (W, H) + + scale = min(W / src_w, H / src_h) + content_h = round(src_h * scale) + got = _subject_height(im) + assert abs(got - ratio * content_h) <= 0.08 * content_h, ( + f"{src_w}x{src_h} alpha={alpha}: 主体 {got}px，内容区高 {content_h}px，" + f"期望约 {ratio * content_h:.0f}px；只缩不放会得到约 {ratio * src_h:.0f}px" + ) + # 与"不拉伸"是两条独立约束：放大到了也可能是拉伸放大的。 + assert scale > 1, "本用例的输入都小于画布，否则测不到放大" + + +def test_transparent_first_frame_background_does_not_depend_on_undefined_rgb(): + """透明像素的 RGB 是未定义值，直接 ``convert("RGB")`` 会把它当真。 + + 抠图后透明区的 RGB 恰好是 0，于是视频输入静默变成黑底；换个抠图实现就换个底色。 + 合成规则必须是声明出来的常量，且同一 alpha、不同未定义 RGB 必须给出同一张图。 + """ + import numpy as _np + + from windup_framework.providers.sufy import _FIRST_FRAME_BG + + black_void = _submitted_first_frame(_sprite(256, 256, alpha=True, void_rgb=(0, 0, 0))) + red_void = _submitted_first_frame(_sprite(256, 256, alpha=True, void_rgb=(255, 0, 0))) + + corner = _np.asarray(black_void)[4, 4] + assert _np.allclose(corner, _FIRST_FRAME_BG, atol=12), f"角落底色 {corner}，应为声明的 {_FIRST_FRAME_BG}" + assert not _np.allclose(corner, (0, 0, 0), atol=12), "透明背景又变成黑底了" + + diff = _np.abs(_np.asarray(black_void, dtype=float) - _np.asarray(red_void, dtype=float)) + assert diff.max() <= 12, f"未定义 RGB 换个值就产出不同图像（最大差 {diff.max()}）" + + +def test_opaque_first_frame_keeps_sampling_its_own_corner_for_padding(): + """不透明输入的补边色沿用角点色 —— 强行改成固定底色会在画面与补边之间造出一条缝。""" + import numpy as _np + + im = _submitted_first_frame(_sprite(256, 256, alpha=False)) + corner = _np.asarray(im)[4, 4] + assert _np.allclose(corner, (200, 200, 200), atol=12), f"补边色 {corner}，应沿用源图角点色" + + +def test_square_first_frame_forms_a_720x720_content_region_in_a_1280x720_canvas(): + """方形输入在 1280x720 里应是 720x720 的等比内容区，左右各补 280px。 + + 与"主体占幅"是两条判据:主体占幅对了也可能是内容区偏了(比如贴在角上)。 + 源图最外一圈填成不透明亮色,内容区边界才量得到 —— 补边色与合成底色相同, + 只看底色是量不出边界的。 + """ + import io as _io + + import numpy as _np + from PIL import Image as _Image + + src = _Image.new("RGBA", (256, 256), (0, 0, 0, 0)) + src.paste((240, 240, 240, 255), (0, 0, 256, 256)) + src.paste((10, 10, 10, 255), (60, 60, 196, 196)) # 内部暗块，避免整幅纯色 + buf = _io.BytesIO(); src.save(buf, "PNG") + + im = _submitted_first_frame(buf.getvalue()) + a = _np.asarray(im.convert("L")) + cols = _np.where((a > 200).any(axis=0))[0] + rows = _np.where((a > 200).any(axis=1))[0] + assert 715 <= cols.max()-cols.min()+1 <= 725, f"内容区宽 {cols.max()-cols.min()+1}，应≈720" + assert 715 <= rows.max()-rows.min()+1 <= 725, f"内容区高 {rows.max()-rows.min()+1}，应≈720" + assert abs(cols.min() - 280) <= 4, f"内容区左边界 {cols.min()}，应≈280（左右各补 280）" From 293b3480b5c8323a147eef454ad877c1596d1e27 Mon Sep 17 00:00:00 2001 From: Johnny Zhang Date: Fri, 21 Aug 2026 14:05:30 +0800 Subject: [PATCH 2/2] =?UTF-8?q?style(tests):=20=E6=8B=86=E5=BC=80=E9=A6=96?= =?UTF-8?q?=E5=B8=A7=E5=86=85=E5=AE=B9=E5=8C=BA=E7=94=A8=E4=BE=8B=E9=87=8C?= =?UTF-8?q?=E7=9A=84=E5=88=86=E5=8F=B7=E8=AF=AD=E5=8F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/tests/test_sufy_video_download.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/tests/test_sufy_video_download.py b/backend/tests/test_sufy_video_download.py index 69c90a1d..fc5e8013 100644 --- a/backend/tests/test_sufy_video_download.py +++ b/backend/tests/test_sufy_video_download.py @@ -1004,7 +1004,8 @@ def test_square_first_frame_forms_a_720x720_content_region_in_a_1280x720_canvas( src = _Image.new("RGBA", (256, 256), (0, 0, 0, 0)) src.paste((240, 240, 240, 255), (0, 0, 256, 256)) src.paste((10, 10, 10, 255), (60, 60, 196, 196)) # 内部暗块，避免整幅纯色 - buf = _io.BytesIO(); src.save(buf, "PNG") + buf = _io.BytesIO() + src.save(buf, "PNG") im = _submitted_first_frame(buf.getvalue()) a = _np.asarray(im.convert("L"))