From 0e447ae57465430fb6fcca49ac3d3347db185eaa Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Mon, 20 Jul 2026 14:47:13 +1000 Subject: [PATCH] fix: catch all exceptions from custom kernel load, not just RuntimeError The model-load phase in eval_kernel_against_ref only caught RuntimeError, so a generated kernel whose __init__ raised anything else (ValueError, TypeError, KeyError, AssertionError from the assert hasattr(...) check, etc.) propagated out of the function instead of being recorded as a graded outcome. The neighbouring phases already treat any exception from the generated code as a graded result: compilation catches Exception -> compiled=False, and correctness catches Exception -> compiled=True, correctness=False. The model-load phase was the only gap between them. Broadening the guard to except Exception matches the sibling phases and means the generated kernel's own faulty __init__ is graded (compiled=True, correctness=False) rather than surfacing as an unhandled exception indistinguishable from an environment failure. metadata["runtime_error_name"] (via get_error_name) already preserves the exception class, so RuntimeError vs. other exception types remains distinguishable in the result metadata. Fixes #159 Co-Authored-By: Claude Sonnet 5 --- src/kernelbench/eval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kernelbench/eval.py b/src/kernelbench/eval.py index 3557dc948f8611..6f7668cf73c995 100644 --- a/src/kernelbench/eval.py +++ b/src/kernelbench/eval.py @@ -566,7 +566,7 @@ def eval_kernel_against_ref( torch.cuda.synchronize(device=device) if verbose: print("[Eval] New Model with Custom CUDA Kernel Loaded") - except RuntimeError as e: + except Exception as e: print( f"Failed to load custom CUDA kernel; Compiled but not able to run, count as runtime error. \nError: {e}" )