From 105015171996cb4662d90bcc044af82ec80de302 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Tue, 14 Jul 2026 16:11:08 -0700 Subject: [PATCH 01/12] Add PackedSimd load/store mappings to table --- src/coreclr/jit/codegenwasm.cpp | 11 ++--------- src/coreclr/jit/hwintrinsiclistwasm.h | 10 +++++----- src/coreclr/jit/hwintrinsicwasm.cpp | 8 -------- 3 files changed, 7 insertions(+), 22 deletions(-) diff --git a/src/coreclr/jit/codegenwasm.cpp b/src/coreclr/jit/codegenwasm.cpp index 1c00c386259e20..85f4ff65d7f52d 100644 --- a/src/coreclr/jit/codegenwasm.cpp +++ b/src/coreclr/jit/codegenwasm.cpp @@ -2747,15 +2747,8 @@ void CodeGen::genCodeForStoreInd(GenTreeStoreInd* tree) } else // A normal store, not a WriteBarrier store { - var_types type = tree->TypeGet(); - if (type == TYP_SIMD16) - { - // Storing a SIMD16 value emits v128.store, but the data operand is not - // materialized as a v128 (it comes through as an i32), producing an invalid - // module. Bail until SIMD16 store is properly supported. - NYI_WASM_SIMD("SIMD16 store indirect"); - } - instruction ins = ins_Store(type); + var_types type = tree->TypeGet(); + instruction ins = ins_Store(type); // TODO-WASM: Memory barriers diff --git a/src/coreclr/jit/hwintrinsiclistwasm.h b/src/coreclr/jit/hwintrinsiclistwasm.h index 3f48197b9d1f53..9892035f9b7c90 100644 --- a/src/coreclr/jit/hwintrinsiclistwasm.h +++ b/src/coreclr/jit/hwintrinsiclistwasm.h @@ -44,11 +44,11 @@ HARDWARE_INTRINSIC(PackedSimd, Divide, HARDWARE_INTRINSIC(PackedSimd, Dot, 16, 2, INS_invalid, INS_invalid, INS_i32x4_dot_i16x8_s, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, ExtractScalar, 16, 2, INS_i8x16_extract_lane_s, INS_i8x16_extract_lane_u, INS_i16x8_extract_lane_s, INS_i16x8_extract_lane_u, INS_i32x4_extract_lane, INS_i32x4_extract_lane, INS_i64x2_extract_lane, INS_i64x2_extract_lane, INS_f32x4_extract_lane, INS_f64x2_extract_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_floor, INS_f64x2_floor, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndInsert, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndSplatVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, LoadScalarVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndInsert, 16, 3, INS_v128_load8_lane, INS_v128_load8_lane, INS_v128_load16_lane, INS_v128_load16_lane, INS_v128_load32_lane, INS_v128_load32_lane, INS_v128_load64_lane, INS_v128_load64_lane, INS_v128_load32_lane, INS_v128_load64_lane, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndSplatVector128, 16, 1, INS_v128_load8_splat, INS_v128_load8_splat, INS_v128_load16_splat, INS_v128_load16_splat, INS_v128_load32_splat, INS_v128_load32_splat INS_v128_load64_splat, INS_v128_load64_splat, INS_v128_load32_splat, INS_v128_load64_splat, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, LoadScalarVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_v128_load32_zero, INS_v128_load32_zero, INS_v128_load64_zero, INS_v128_load64_zero, INS_v128_load32_zero, INS_v128_load64_zero, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, LoadVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, LoadWideningVector128, 16, 1, INS_v128_load8x8_s, INS_v128_load8x8_u, INS_v128_load16x4_s, INS_v128_load16x4_u, INS_v128_load32x2_s, INS_v128_load32x2_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, LoadWideningVector128, 16, 1, INS_v128_load8x8_s, INS_v128_load8x8_u, INS_v128_load16x4_s, INS_v128_load16x4_u, INS_v128_load32x2_s, INS_v128_load32x2_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Max, 16, 2, INS_i8x16_max_s, INS_i8x16_max_u, INS_i16x8_max_s, INS_i16x8_max_u, INS_i32x4_max_s, INS_i32x4_max_u, INS_invalid, INS_invalid, INS_f32x4_max, INS_f64x2_max, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) HARDWARE_INTRINSIC(PackedSimd, Min, 16, 2, INS_i8x16_min_s, INS_i8x16_min_u, INS_i16x8_min_s, INS_i16x8_min_u, INS_i32x4_min_s, INS_i32x4_min_u, INS_invalid, INS_invalid, INS_f32x4_min, INS_f64x2_min, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) HARDWARE_INTRINSIC(PackedSimd, Multiply, 16, 2, INS_invalid, INS_invalid, INS_i16x8_mul, INS_i16x8_mul, INS_i32x4_mul, INS_i32x4_mul, INS_i64x2_mul, INS_i64x2_mul, INS_f32x4_mul, INS_f64x2_mul, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) @@ -72,7 +72,7 @@ HARDWARE_INTRINSIC(PackedSimd, SignExtendWideningUpper, HARDWARE_INTRINSIC(PackedSimd, Splat, 16, 1, INS_i8x16_splat, INS_i8x16_splat, INS_i16x8_splat, INS_i16x8_splat, INS_i32x4_splat, INS_i32x4_splat, INS_i64x2_splat, INS_i64x2_splat, INS_f32x4_splat, INS_f64x2_splat, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Sqrt, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_sqrt, INS_f64x2_sqrt, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Store, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(PackedSimd, StoreSelectedScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(PackedSimd, StoreSelectedScalar, 16, 3, INS_v128_store8_lane, INS_v128_store8_lane, INS_v128_store16_lane, INS_v128_store16_lane, INS_v128_store32_lane, INS_v128_store32_lane, INS_v128_store64_lane, INS_v128_store64_lane, INS_v128_store32_lane, INS_v128_store64_lane, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg) HARDWARE_INTRINSIC(PackedSimd, Subtract, 16, 2, INS_i8x16_sub, INS_i8x16_sub, INS_i16x8_sub, INS_i16x8_sub, INS_i32x4_sub, INS_i32x4_sub, INS_i64x2_sub, INS_i64x2_sub, INS_f32x4_sub, INS_f64x2_sub, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, SubtractSaturate, 16, 2, INS_i8x16_sub_sat_s, INS_i8x16_sub_sat_u, INS_i16x8_sub_sat_s, INS_i16x8_sub_sat_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Swizzle, 16, 2, INS_i8x16_swizzle, INS_i8x16_swizzle, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) diff --git a/src/coreclr/jit/hwintrinsicwasm.cpp b/src/coreclr/jit/hwintrinsicwasm.cpp index 9e5fe0b4ec45f7..4db20a60d05a83 100644 --- a/src/coreclr/jit/hwintrinsicwasm.cpp +++ b/src/coreclr/jit/hwintrinsicwasm.cpp @@ -170,14 +170,6 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } - case NI_PackedSimd_LoadScalarVector128: - case NI_PackedSimd_LoadScalarAndSplatVector128: - case NI_PackedSimd_LoadScalarAndInsert: - case NI_PackedSimd_LoadWideningVector128: - { - break; - } - case NI_PackedSimd_Store: { assert(sig->numArgs == 2); From 45f618eeba2039c018a5dd5c3e72285c5864aa17 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 15 Jul 2026 09:54:18 -0700 Subject: [PATCH 02/12] Fix OperIsMemory{Load,Store} for PackedSimd operations --- src/coreclr/jit/gentree.cpp | 8 ++++---- src/coreclr/jit/hwintrinsiclistwasm.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index bbec675a6f07a1..4bb94a8b30ddf9 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -30252,7 +30252,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const { GenTree* addr = nullptr; -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) +#if defined(TARGET_XARCH) || defined(TARGET_ARM64) || defined(TARGET_WASM) NamedIntrinsic intrinsicId = GetHWIntrinsicId(); HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(intrinsicId); @@ -30442,7 +30442,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const } } #endif // TARGET_XARCH -#endif // TARGET_XARCH || TARGET_ARM64 +#endif // TARGET_XARCH || TARGET_ARM64 || TARGET_WASM if (pAddr != nullptr) { @@ -30509,7 +30509,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryStore(GenTree** pAddr) const { GenTree* addr = nullptr; -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) +#if defined(TARGET_XARCH) || defined(TARGET_ARM64) || defined(TARGET_WASM) NamedIntrinsic intrinsicId = GetHWIntrinsicId(); HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(intrinsicId); @@ -30582,7 +30582,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryStore(GenTree** pAddr) const } } #endif // TARGET_XARCH -#endif // TARGET_XARCH || TARGET_ARM64 +#endif // TARGET_XARCH || TARGET_ARM64 || TARGET_WASM if (pAddr != nullptr) { diff --git a/src/coreclr/jit/hwintrinsiclistwasm.h b/src/coreclr/jit/hwintrinsiclistwasm.h index 9892035f9b7c90..f2493480c30047 100644 --- a/src/coreclr/jit/hwintrinsiclistwasm.h +++ b/src/coreclr/jit/hwintrinsiclistwasm.h @@ -45,7 +45,7 @@ HARDWARE_INTRINSIC(PackedSimd, Dot, HARDWARE_INTRINSIC(PackedSimd, ExtractScalar, 16, 2, INS_i8x16_extract_lane_s, INS_i8x16_extract_lane_u, INS_i16x8_extract_lane_s, INS_i16x8_extract_lane_u, INS_i32x4_extract_lane, INS_i32x4_extract_lane, INS_i64x2_extract_lane, INS_i64x2_extract_lane, INS_f32x4_extract_lane, INS_f64x2_extract_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_floor, INS_f64x2_floor, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndInsert, 16, 3, INS_v128_load8_lane, INS_v128_load8_lane, INS_v128_load16_lane, INS_v128_load16_lane, INS_v128_load32_lane, INS_v128_load32_lane, INS_v128_load64_lane, INS_v128_load64_lane, INS_v128_load32_lane, INS_v128_load64_lane, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndSplatVector128, 16, 1, INS_v128_load8_splat, INS_v128_load8_splat, INS_v128_load16_splat, INS_v128_load16_splat, INS_v128_load32_splat, INS_v128_load32_splat INS_v128_load64_splat, INS_v128_load64_splat, INS_v128_load32_splat, INS_v128_load64_splat, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndSplatVector128, 16, 1, INS_v128_load8_splat, INS_v128_load8_splat, INS_v128_load16_splat, INS_v128_load16_splat, INS_v128_load32_splat, INS_v128_load32_splat, INS_v128_load64_splat, INS_v128_load64_splat, INS_v128_load32_splat, INS_v128_load64_splat, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, LoadScalarVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_v128_load32_zero, INS_v128_load32_zero, INS_v128_load64_zero, INS_v128_load64_zero, INS_v128_load32_zero, INS_v128_load64_zero, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, LoadVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(PackedSimd, LoadWideningVector128, 16, 1, INS_v128_load8x8_s, INS_v128_load8x8_u, INS_v128_load16x4_s, INS_v128_load16x4_u, INS_v128_load32x2_s, INS_v128_load32x2_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) From 7fd077ac967b5109f7b1b99800a42cd1c20a39cf Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Wed, 15 Jul 2026 13:27:14 -0700 Subject: [PATCH 03/12] Implement SIMD element-wise load/store codegen --- src/coreclr/jit/hwintrinsic.h | 13 +++++++-- src/coreclr/jit/hwintrinsiccodegenwasm.cpp | 32 +++++++++++++++++++++- src/coreclr/jit/hwintrinsiclistwasm.h | 8 +++--- src/coreclr/jit/lowerwasm.cpp | 32 ++++++++++++---------- 4 files changed, 63 insertions(+), 22 deletions(-) diff --git a/src/coreclr/jit/hwintrinsic.h b/src/coreclr/jit/hwintrinsic.h index 0b7e966a5e4bf0..b98d83c6c4a289 100644 --- a/src/coreclr/jit/hwintrinsic.h +++ b/src/coreclr/jit/hwintrinsic.h @@ -264,6 +264,8 @@ enum HWIntrinsicFlag : uint64_t // The intrinsic supports some sort of containment analysis HW_Flag_SupportsContainment = 0x400, HW_Flag_ReturnsPerElementMask = 0x800, + // The intrinsic has a required immediate operand + HW_Flag_HasImmediateOperand = 0x1000, #else #error Unsupported platform #endif @@ -1003,10 +1005,10 @@ struct HWIntrinsicInfo static bool HasImmediateOperand(NamedIntrinsic id) { -#if defined(TARGET_ARM64) +#if defined(TARGET_ARM64) || defined(TARGET_WASM) const HWIntrinsicFlag flags = lookupFlags(id); return ((flags & HW_Flag_HasImmediateOperand) != 0); -#elif defined(TARGET_XARCH) || defined(TARGET_WASM) +#elif defined(TARGET_XARCH) return lookupCategory(id) == HW_Category_IMM; #else return false; @@ -1472,7 +1474,12 @@ struct HWIntrinsic final inline bool needsJumpTableFallback() const { - return !m_node->GetImmOp()->IsCnsIntOrI(); + if (HWIntrinsicInfo::HasImmediateOperand(id)) + { + return !m_node->GetImmOp()->IsCnsIntOrI(); + } + + return false; } uint8_t GetImmediateLaneOperand() const diff --git a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp index d9d47e7914db7f..39ad759d018829 100644 --- a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp @@ -55,6 +55,25 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) } break; } + case HW_Category_MemoryStore: + case HW_Category_MemoryLoad: + { + emitAttr elemSize = emitActualTypeSize(node->GetSimdBaseType()); + if (info.needsJumpTableFallback()) + { + genHWIntrinsicJumpTableFallback(node, info); + } + else if (HWIntrinsicInfo::HasImmediateOperand(info.id)) + { + GetEmitter()->emitIns_MemargLane(ins, elemSize, 0, info.GetImmediateLaneOperand()); + } + else + { + GetEmitter()->emitIns_I(ins, elemSize, 0); + } + + break; + } default: { NYI_WASM_SIMD("CodeGen::genHWIntrinsic: Unsupported category for table-driven intrinsic"); @@ -111,7 +130,11 @@ void CodeGen::genHWIntrinsicJumpTableFallback(GenTreeHWIntrinsic* node, HWIntrin int simdSize = node->GetSimdSize(); instruction const ins = HWIntrinsicInfo::lookupIns(info.id, info.baseType, m_compiler); int immUpperBound = HWIntrinsicInfo::lookupImmUpperBound(info.id, simdSize, info.baseType); - WasmValueType resultType = ActualTypeToWasmValueType(genActualType(node->TypeGet())); + WasmValueType resultType = WasmValueType::Invalid; + if (!node->TypeIs(TYP_VOID)) + { + resultType = ActualTypeToWasmValueType(genActualType(node->TypeGet())); + } GenTree* immOp = node->GetImmOp(); regNumber immReg = GetMultiUseOperandReg(immOp); @@ -177,6 +200,13 @@ void CodeGen::genHWIntrinsicJumpTableFallback(GenTreeHWIntrinsic* node, HWIntrin GetEmitter()->emitIns_Lane(ins, static_cast(i)); break; } + case HW_Category_MemoryLoad: + case HW_Category_MemoryStore: + { + emitAttr elemSize = emitActualTypeSize(node->GetSimdBaseType()); + GetEmitter()->emitIns_MemargLane(ins, elemSize, 0, static_cast(i)); + break; + } default: { NYI_WASM_SIMD( diff --git a/src/coreclr/jit/hwintrinsiclistwasm.h b/src/coreclr/jit/hwintrinsiclistwasm.h index f2493480c30047..1c5385f7a9522a 100644 --- a/src/coreclr/jit/hwintrinsiclistwasm.h +++ b/src/coreclr/jit/hwintrinsiclistwasm.h @@ -42,9 +42,9 @@ HARDWARE_INTRINSIC(PackedSimd, ConvertToSingle, HARDWARE_INTRINSIC(PackedSimd, ConvertToUInt32Saturate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_i32x4_trunc_sat_u_f32x4, INS_i32x4_trunc_sat_u_f64x2_zero, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Divide, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_div, INS_f64x2_div, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Dot, 16, 2, INS_invalid, INS_invalid, INS_i32x4_dot_i16x8_s, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ExtractScalar, 16, 2, INS_i8x16_extract_lane_s, INS_i8x16_extract_lane_u, INS_i16x8_extract_lane_s, INS_i16x8_extract_lane_u, INS_i32x4_extract_lane, INS_i32x4_extract_lane, INS_i64x2_extract_lane, INS_i64x2_extract_lane, INS_f32x4_extract_lane, INS_f64x2_extract_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ExtractScalar, 16, 2, INS_i8x16_extract_lane_s, INS_i8x16_extract_lane_u, INS_i16x8_extract_lane_s, INS_i16x8_extract_lane_u, INS_i32x4_extract_lane, INS_i32x4_extract_lane, INS_i64x2_extract_lane, INS_i64x2_extract_lane, INS_f32x4_extract_lane, INS_f64x2_extract_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) HARDWARE_INTRINSIC(PackedSimd, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_floor, INS_f64x2_floor, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndInsert, 16, 3, INS_v128_load8_lane, INS_v128_load8_lane, INS_v128_load16_lane, INS_v128_load16_lane, INS_v128_load32_lane, INS_v128_load32_lane, INS_v128_load64_lane, INS_v128_load64_lane, INS_v128_load32_lane, INS_v128_load64_lane, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndInsert, 16, 3, INS_v128_load8_lane, INS_v128_load8_lane, INS_v128_load16_lane, INS_v128_load16_lane, INS_v128_load32_lane, INS_v128_load32_lane, INS_v128_load64_lane, INS_v128_load64_lane, INS_v128_load32_lane, INS_v128_load64_lane, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand) HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndSplatVector128, 16, 1, INS_v128_load8_splat, INS_v128_load8_splat, INS_v128_load16_splat, INS_v128_load16_splat, INS_v128_load32_splat, INS_v128_load32_splat, INS_v128_load64_splat, INS_v128_load64_splat, INS_v128_load32_splat, INS_v128_load64_splat, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, LoadScalarVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_v128_load32_zero, INS_v128_load32_zero, INS_v128_load64_zero, INS_v128_load64_zero, INS_v128_load32_zero, INS_v128_load64_zero, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, LoadVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) @@ -61,7 +61,7 @@ HARDWARE_INTRINSIC(PackedSimd, Or, HARDWARE_INTRINSIC(PackedSimd, PopCount, 16, 1, INS_invalid, INS_i8x16_popcnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, PseudoMax, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_pmax, INS_f64x2_pmax, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, PseudoMin, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_pmin, INS_f64x2_pmin, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ReplaceScalar, 16, 3, INS_i8x16_replace_lane, INS_i8x16_replace_lane, INS_i16x8_replace_lane, INS_i16x8_replace_lane, INS_i32x4_replace_lane, INS_i32x4_replace_lane, INS_i64x2_replace_lane, INS_i64x2_replace_lane, INS_f32x4_replace_lane, INS_f64x2_replace_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ReplaceScalar, 16, 3, INS_i8x16_replace_lane, INS_i8x16_replace_lane, INS_i16x8_replace_lane, INS_i16x8_replace_lane, INS_i32x4_replace_lane, INS_i32x4_replace_lane, INS_i64x2_replace_lane, INS_i64x2_replace_lane, INS_f32x4_replace_lane, INS_f64x2_replace_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) HARDWARE_INTRINSIC(PackedSimd, RoundToNearest, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_nearest, INS_f64x2_nearest, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, ShiftLeft, 16, 2, INS_i8x16_shl, INS_i8x16_shl, INS_i16x8_shl, INS_i16x8_shl, INS_i32x4_shl, INS_i32x4_shl, INS_i64x2_shl, INS_i64x2_shl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, ShiftRightArithmetic, 16, 2, INS_i8x16_shr_s, INS_i8x16_shr_s, INS_i16x8_shr_s, INS_i16x8_shr_s, INS_i32x4_shr_s, INS_i32x4_shr_s, INS_i64x2_shr_s, INS_i64x2_shr_s, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) @@ -72,7 +72,7 @@ HARDWARE_INTRINSIC(PackedSimd, SignExtendWideningUpper, HARDWARE_INTRINSIC(PackedSimd, Splat, 16, 1, INS_i8x16_splat, INS_i8x16_splat, INS_i16x8_splat, INS_i16x8_splat, INS_i32x4_splat, INS_i32x4_splat, INS_i64x2_splat, INS_i64x2_splat, INS_f32x4_splat, INS_f64x2_splat, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Sqrt, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_sqrt, INS_f64x2_sqrt, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Store, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(PackedSimd, StoreSelectedScalar, 16, 3, INS_v128_store8_lane, INS_v128_store8_lane, INS_v128_store16_lane, INS_v128_store16_lane, INS_v128_store32_lane, INS_v128_store32_lane, INS_v128_store64_lane, INS_v128_store64_lane, INS_v128_store32_lane, INS_v128_store64_lane, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(PackedSimd, StoreSelectedScalar, 16, 3, INS_v128_store8_lane, INS_v128_store8_lane, INS_v128_store16_lane, INS_v128_store16_lane, INS_v128_store32_lane, INS_v128_store32_lane, INS_v128_store64_lane, INS_v128_store64_lane, INS_v128_store32_lane, INS_v128_store64_lane, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand) HARDWARE_INTRINSIC(PackedSimd, Subtract, 16, 2, INS_i8x16_sub, INS_i8x16_sub, INS_i16x8_sub, INS_i16x8_sub, INS_i32x4_sub, INS_i32x4_sub, INS_i64x2_sub, INS_i64x2_sub, INS_f32x4_sub, INS_f64x2_sub, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, SubtractSaturate, 16, 2, INS_i8x16_sub_sat_s, INS_i8x16_sub_sat_u, INS_i16x8_sub_sat_s, INS_i16x8_sub_sat_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Swizzle, 16, 2, INS_i8x16_swizzle, INS_i8x16_swizzle, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index da8e304283aed5..5f611e7da9e2da 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -828,6 +828,7 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) { NamedIntrinsic intrinsic = node->GetHWIntrinsicId(); HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(intrinsic); + bool hasImmOp = HWIntrinsicInfo::HasImmediateOperand(intrinsic); switch (intrinsic) { @@ -868,11 +869,22 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) case NI_PackedSimd_ExtractScalar: case NI_PackedSimd_ReplaceScalar: + case NI_PackedSimd_LoadScalarAndInsert: + case NI_PackedSimd_StoreSelectedScalar: { - assert(category == HW_Category_IMM); + assert(hasImmOp); return LowerHWIntrinsicWithImm(node); } + case NI_PackedSimd_LoadScalarAndSplatVector128: + case NI_PackedSimd_LoadScalarVector128: + case NI_PackedSimd_LoadWideningVector128: + { + // These intrinsics don't a required immediate operand + assert(!hasImmOp); + break; + } + default: { assert(category == HW_Category_SIMD); @@ -1210,21 +1222,13 @@ GenTree* Lowering::LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node) // void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) { - HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(node->GetHWIntrinsicId()); - switch (category) + NamedIntrinsic intrinsicId = node->GetHWIntrinsicId(); + if (HWIntrinsicInfo::HasImmediateOperand(intrinsicId)) { - case HWIntrinsicCategory::HW_Category_IMM: + GenTree* immOp = node->GetImmOp(); + if (immOp->IsCnsIntOrI()) { - GenTree* immOp = node->GetImmOp(); - if (immOp->IsCnsIntOrI()) - { - MakeSrcContained(node, immOp); - } - break; - } - default: - { - break; + MakeSrcContained(node, immOp); } } } From f9504c5450b4cec032fad06fb98682845f2cbf72 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Thu, 16 Jul 2026 17:56:34 -0700 Subject: [PATCH 04/12] Properly lower PackedSimd.Shuffle with non-constant operands --- src/coreclr/jit/hwintrinsic.h | 13 +++ src/coreclr/jit/hwintrinsiccodegenwasm.cpp | 13 ++- src/coreclr/jit/lower.h | 1 + src/coreclr/jit/lowerwasm.cpp | 93 +++++++++++++++++++++- 4 files changed, 114 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/hwintrinsic.h b/src/coreclr/jit/hwintrinsic.h index b98d83c6c4a289..7ba21d056f02a1 100644 --- a/src/coreclr/jit/hwintrinsic.h +++ b/src/coreclr/jit/hwintrinsic.h @@ -1331,6 +1331,12 @@ struct HWIntrinsicInfo *imm1Pos = 3; break; } + case NI_PackedSimd_Shuffle: + { + // (v128, v128, lane_imm) + *imm1Pos = 3; + break; + } default: { unreached(); @@ -1495,6 +1501,13 @@ struct HWIntrinsic final return static_cast(lane); } + simd16_t GetImmediateVecOperand() const + { + GenTree* immOp = m_node->GetImmOp(); + assert(immOp->IsCnsVec()); + return immOp->AsVecCon()->gtSimdVal; + } + NamedIntrinsic id; HWIntrinsicCategory category; GenTree* op1; diff --git a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp index 39ad759d018829..7c942ec3df93b0 100644 --- a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp @@ -26,9 +26,6 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX // void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { - // emitIns_Lane - // emitIns_Memarg_Lane - const HWIntrinsic info(node); genConsumeMultiOpOperands(node); @@ -40,7 +37,15 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { case HW_Category_SIMD: { - GetEmitter()->emitIns(ins); + if (info.id == NI_PackedSimd_Shuffle) + { + simd16_t shuffleMask = info.GetImmediateVecOperand(); + GetEmitter()->emitIns_V128Imm(ins, shuffleMask.u8); + } + else + { + GetEmitter()->emitIns(ins); + } break; } case HW_Category_IMM: diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index 2926e1c2477562..8f899884975899 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -523,6 +523,7 @@ class Lowering final : public Phase #elif defined(TARGET_WASM) GenTree* LowerHWIntrinsicCompareUnsignedLong(GenTreeHWIntrinsic* node); GenTree* LowerHWIntrinsicWithImm(GenTreeHWIntrinsic* node); + GenTree* LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node); #endif // !TARGET_XARCH && !TARGET_ARM64 GenTree* InsertNewSimdCreateScalarUnsafeNode(var_types type, GenTree* op1, diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index 5f611e7da9e2da..ef2a01bbd12767 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -876,6 +876,11 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) return LowerHWIntrinsicWithImm(node); } + case NI_PackedSimd_Shuffle: + { + return LowerHWIntrinsicNativeShuffle(node); + } + case NI_PackedSimd_LoadScalarAndSplatVector128: case NI_PackedSimd_LoadScalarVector128: case NI_PackedSimd_LoadWideningVector128: @@ -1214,6 +1219,87 @@ GenTree* Lowering::LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node) return LowerNode(node); } +GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) +{ + assert(node->GetHWIntrinsicId() == NI_PackedSimd_Shuffle); + + GenTree* op1 = node->Op(1); + GenTree* op2 = node->Op(2); + GenTree* shuffleMask = node->Op(3); + var_types resultType = node->TypeGet(); + + // No extra work to do if the shuffle is a constant vector, it can be contained as an immediate and emitted. + if (shuffleMask->IsCnsVec()) + { + ContainCheckHWIntrinsic(node); + return node->gtNext; + } + + // If the shuffle mask is not a constant vector, we will need to rewrite the shuffle into two swizzles: + // 1 to handle elements from the first vector, and 1 to handle elements from the second vector. The two swizzles will then be or'd together to produce the final result. + // We will be constructing IR like the following: + // /--* op1 simd + // +--* originalMask simd + // tmp1 = * HWINTRINSIC simd byte PackedSimd.Swizzle + // upperBnd = * CNS_VEC simd byte <0x10, 0x10, ...> + // /--* originalMask simd + // +--* upperBnd simd + // upperMask = * HWINRINSIC simd byte PackedSimd.Subtract + // /--* op2 simd + // +--* upperMask simd + // tmp2 = * HWINTRINSIC simd byte PackedSimd.Swizzle + // /--* tmp1 simd + // +--* tmp2 simd + // res = * HWINTRINSIC simd byte PackedSimd.Or + + + // Shuffle mask will be used twice, replace with a local + LIR::Use shuffleMaskUse(BlockRange(), &node->Op(3), node); + unsigned int shuffleMaskTmp = shuffleMaskUse.ReplaceWithLclVar(m_compiler); + + // Do a swizzle of the first vector with the original shuffle mask (now loaded from a local), which will produce a vector with the elements from the first vector in the correct order, and zero's for all elements which correspond to the second vector. + GenTree* swizzle1 = m_compiler->gtNewSimdHWIntrinsicNode(resultType, op1, node->Op(3), NI_PackedSimd_Swizzle, TYP_BYTE, 16); + BlockRange().InsertBefore(node, swizzle1); + LowerNode(swizzle1); + + // Subtract 16 from each mask element to mark each element which corresponds to the upper vector as unused, leading to a zero in the result of the swizzle. + GenTreeVecCon* upperBound = m_compiler->gtNewVconNode(shuffleMask->TypeGet()); + upperBound->EvaluateBroadcastInPlace(TYP_BYTE, static_cast(16)); + BlockRange().InsertBefore(node, upperBound); + LowerNode(upperBound); + + // Re-load the original shuffle mask + GenTreeLclVar* shuffleMaskLclVar = m_compiler->gtNewLclVarNode(shuffleMaskTmp, shuffleMask->TypeGet()); + BlockRange().InsertBefore(node, shuffleMaskLclVar); + LowerNode(shuffleMaskLclVar); + + GenTree* upperMask = m_compiler->gtNewSimdHWIntrinsicNode(shuffleMask->TypeGet(), shuffleMaskLclVar, upperBound, NI_PackedSimd_Subtract, TYP_BYTE, 16); + BlockRange().InsertBefore(node, upperMask); + LowerNode(upperMask); + + GenTree* swizzle2 = m_compiler->gtNewSimdHWIntrinsicNode(resultType, op2, upperMask, NI_PackedSimd_Swizzle, TYP_BYTE, 16); + BlockRange().InsertBefore(node, swizzle2); + LowerNode(swizzle2); + + // Since we've left zero's for all the elements which correspond to the upper vector, we can just or the two swizzles together to get the final result. + GenTreeHWIntrinsic* result = m_compiler->gtNewSimdHWIntrinsicNode(resultType, swizzle1, swizzle2, NI_PackedSimd_Or, TYP_INT, 16); + BlockRange().InsertBefore(node, result); + + LIR::Use use; + if (BlockRange().TryGetUse(node, &use)) + { + use.ReplaceWith(result); + } + else + { + result->SetUnusedValue(); + } + + BlockRange().Remove(node); + + return LowerNode(result); +} + //---------------------------------------------------------------------------------------------- // ContainCheckHWIntrinsic: Perform containment analysis for a hardware intrinsic node. // @@ -1223,10 +1309,13 @@ GenTree* Lowering::LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node) void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) { NamedIntrinsic intrinsicId = node->GetHWIntrinsicId(); - if (HWIntrinsicInfo::HasImmediateOperand(intrinsicId)) + if (HWIntrinsicInfo::HasImmediateOperand(intrinsicId) || node->GetHWIntrinsicId() == NI_PackedSimd_Shuffle) { GenTree* immOp = node->GetImmOp(); - if (immOp->IsCnsIntOrI()) + // An immediate operand for SIMD should either be: + // - An integer lane index OR + // - A constant vector (for example, for a shuffle) + if (immOp->IsCnsIntOrI() || immOp->IsCnsVec()) { MakeSrcContained(node, immOp); } From 8d07863d4bb092238c84846fc549876676e917ca Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Thu, 16 Jul 2026 18:39:51 -0700 Subject: [PATCH 05/12] Enable null checks for PackedSimd loads/stores --- src/coreclr/jit/hwintrinsiccodegenwasm.cpp | 7 +++++++ src/coreclr/jit/lowerwasm.cpp | 6 ++++++ src/coreclr/jit/stacklevelsetter.cpp | 12 ++++++++++++ 3 files changed, 25 insertions(+) diff --git a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp index 39ad759d018829..07bed6302a7f86 100644 --- a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp @@ -59,6 +59,13 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) case HW_Category_MemoryLoad: { emitAttr elemSize = emitActualTypeSize(node->GetSimdBaseType()); + GenTree* addr = nullptr; + assert(node->OperIsMemoryLoad(&addr) || node->OperIsMemoryStore(&addr)); + assert(addr != nullptr); + + regNumber addrReg = GetMultiUseOperandReg(addr); + genEmitNullCheck(addrReg); + if (info.needsJumpTableFallback()) { genHWIntrinsicJumpTableFallback(node, info); diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index 5f611e7da9e2da..bd94ca4869aaf0 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -829,6 +829,12 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) NamedIntrinsic intrinsic = node->GetHWIntrinsicId(); HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(intrinsic); bool hasImmOp = HWIntrinsicInfo::HasImmediateOperand(intrinsic); + GenTree* addr = nullptr; + + if (node->OperIsMemoryLoad(&addr) || node->OperIsMemoryStore(&addr)) + { + SetMultiplyUsed(addr DEBUGARG("LowerHWIntrinsic memory address (null check)")); + } switch (intrinsic) { diff --git a/src/coreclr/jit/stacklevelsetter.cpp b/src/coreclr/jit/stacklevelsetter.cpp index c33de2c033d374..925ae2c8ea8ce4 100644 --- a/src/coreclr/jit/stacklevelsetter.cpp +++ b/src/coreclr/jit/stacklevelsetter.cpp @@ -273,8 +273,20 @@ void StackLevelSetter::SetThrowHelperBlocks(GenTree* node, BasicBlock* block) } } break; +#elif defined(FEATURE_HW_INTRINSICS) && defined(TARGET_WASM) + case GT_HWINTRINSIC: + { + HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(node->AsHWIntrinsic()->GetHWIntrinsicId()); + if (category == HW_Category_MemoryLoad || category == HW_Category_MemoryStore) + { + SetThrowHelperBlock(SCK_NULL_CHECK, block); + } + } + break; + #endif // defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH) + case GT_INDEX_ADDR: if (node->AsIndexAddr()->IsBoundsChecked()) { From 0df646ae10c9ee1b882584f5e62ca825cde093a4 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 17 Jul 2026 10:34:23 -0700 Subject: [PATCH 06/12] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/coreclr/jit/lowerwasm.cpp | 2 +- src/coreclr/jit/stacklevelsetter.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index bd94ca4869aaf0..b89f9324027aaf 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -886,7 +886,7 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) case NI_PackedSimd_LoadScalarVector128: case NI_PackedSimd_LoadWideningVector128: { - // These intrinsics don't a required immediate operand + // These intrinsics don't require an immediate operand assert(!hasImmOp); break; } diff --git a/src/coreclr/jit/stacklevelsetter.cpp b/src/coreclr/jit/stacklevelsetter.cpp index 925ae2c8ea8ce4..47b5fc8f0a733d 100644 --- a/src/coreclr/jit/stacklevelsetter.cpp +++ b/src/coreclr/jit/stacklevelsetter.cpp @@ -284,7 +284,7 @@ void StackLevelSetter::SetThrowHelperBlocks(GenTree* node, BasicBlock* block) } break; -#endif // defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH) +#endif // defined(FEATURE_HW_INTRINSICS) && (defined(TARGET_XARCH) || defined(TARGET_WASM)) case GT_INDEX_ADDR: From 6ddbd32ed8bc387ecea6b4ebda469dc4996cb586 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 17 Jul 2026 10:55:11 -0700 Subject: [PATCH 07/12] Add PackedSimdTests to exercise null check for loads/stores --- .../Wasm/PackedSimd/PackedSimdTests.cs | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs b/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs index db66eb3b89a36c..7ff2ea5b268f2d 100644 --- a/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs +++ b/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs @@ -4,6 +4,7 @@ using System; using System.Diagnostics.CodeAnalysis; using System.Reflection; +using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.Wasm; using Xunit; @@ -491,6 +492,48 @@ public static unsafe void LoadScalarAndSplatTest() Assert.Equal(Vector128.Create(3.14f, 3.14f, 3.14f, 3.14f), floatSplat); } + [Fact] + public static unsafe void LoadStoreNullCheckTest() + { + Assert.Throws(() => LoadScalarAndSplatVector128(null)); + Assert.Throws(() => LoadScalarVector128(null)); + Assert.Throws(() => LoadWideningVector128(null)); + Assert.Throws(() => LoadScalarAndInsert(null, 2)); + Assert.Throws(() => StoreSelectedScalar(null, 2)); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe Vector128 LoadScalarAndSplatVector128(int* address) + { + return PackedSimd.LoadScalarAndSplatVector128(address); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe Vector128 LoadScalarVector128(int* address) + { + return PackedSimd.LoadScalarVector128(address); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe Vector128 LoadWideningVector128(sbyte* address) + { + return PackedSimd.LoadWideningVector128(address); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe Vector128 LoadScalarAndInsert(int* address, byte index) + { + Vector128 vector = Vector128.Create(1, 2, 3, 4); + return PackedSimd.LoadScalarAndInsert(address, vector, index); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe void StoreSelectedScalar(int* address, byte index) + { + Vector128 vector = Vector128.Create(1, 2, 3, 4); + PackedSimd.StoreSelectedScalar(address, vector, index); + } + [Fact] public static unsafe void LoadWideningTest() { From f1f45cec48b134108ece5621931acb45435dc0ee Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 17 Jul 2026 11:36:36 -0700 Subject: [PATCH 08/12] jit-format --- src/coreclr/jit/stacklevelsetter.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/coreclr/jit/stacklevelsetter.cpp b/src/coreclr/jit/stacklevelsetter.cpp index 47b5fc8f0a733d..ccba1b54334f46 100644 --- a/src/coreclr/jit/stacklevelsetter.cpp +++ b/src/coreclr/jit/stacklevelsetter.cpp @@ -286,7 +286,6 @@ void StackLevelSetter::SetThrowHelperBlocks(GenTree* node, BasicBlock* block) #endif // defined(FEATURE_HW_INTRINSICS) && (defined(TARGET_XARCH) || defined(TARGET_WASM)) - case GT_INDEX_ADDR: if (node->AsIndexAddr()->IsBoundsChecked()) { From d53c1cc428bf248b83dd9072121164ba6e9c5f0d Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 17 Jul 2026 11:59:48 -0700 Subject: [PATCH 09/12] Add header comment --- src/coreclr/jit/lowerwasm.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index e89a2f081bedbd..0438a10de105c5 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -1309,6 +1309,15 @@ GenTree* Lowering::LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node) return LowerNode(node); } +// -------------------------------------------------------------------------------- +// LowerHWIntrinsicNativeShuffle: Lowers a PackedSimd Shuffle call with a possibly non-constant mask +// +// Arguments: +// node - The hardware intrinsic node. +// +// Notes: +// If the shuffle mask is a constant vector, it can be contained as an immediate and emitted. Otherwise, +// the shuffle is rewritten into two swizzles for the upper and lower input vectors and combined into the final result. GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) { assert(node->GetHWIntrinsicId() == NI_PackedSimd_Shuffle); From e78535b29e75251d63cd1f53f09336c2ef90caee Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Fri, 17 Jul 2026 17:50:46 -0700 Subject: [PATCH 10/12] Fix segfault in release: move side effecting statement out of assert() --- src/coreclr/jit/hwintrinsiccodegenwasm.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp index ca3125f7fd91f9..a39265cd3609bd 100644 --- a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp @@ -75,8 +75,8 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { emitAttr elemSize = emitActualTypeSize(node->GetSimdBaseType()); GenTree* addr = nullptr; - assert(node->OperIsMemoryLoad(&addr) || node->OperIsMemoryStore(&addr)); - assert(addr != nullptr); + bool isMem = node->OperIsMemoryLoad(&addr) || node->OperIsMemoryStore(&addr); + assert(isMem && addr != nullptr); regNumber addrReg = GetMultiUseOperandReg(addr); genEmitNullCheck(addrReg); From bdbba8c38589e81cc4b2c17e2bbe71c1b9f8aee1 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 20 Jul 2026 09:26:50 -0700 Subject: [PATCH 11/12] jit-format --- src/coreclr/jit/hwintrinsiccodegenwasm.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp index a39265cd3609bd..0bb051dfcfdc13 100644 --- a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp @@ -75,7 +75,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { emitAttr elemSize = emitActualTypeSize(node->GetSimdBaseType()); GenTree* addr = nullptr; - bool isMem = node->OperIsMemoryLoad(&addr) || node->OperIsMemoryStore(&addr); + bool isMem = node->OperIsMemoryLoad(&addr) || node->OperIsMemoryStore(&addr); assert(isMem && addr != nullptr); regNumber addrReg = GetMultiUseOperandReg(addr); From 27bf1040f6d78e5e30c68419953fef06eb8f9005 Mon Sep 17 00:00:00 2001 From: Adam Perlin Date: Mon, 20 Jul 2026 09:53:50 -0700 Subject: [PATCH 12/12] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/coreclr/jit/hwintrinsic.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/coreclr/jit/hwintrinsic.h b/src/coreclr/jit/hwintrinsic.h index 7ba21d056f02a1..dcb7ebe20c8c17 100644 --- a/src/coreclr/jit/hwintrinsic.h +++ b/src/coreclr/jit/hwintrinsic.h @@ -1333,8 +1333,7 @@ struct HWIntrinsicInfo } case NI_PackedSimd_Shuffle: { - // (v128, v128, lane_imm) - *imm1Pos = 3; + // (v128, v128, shuffle_mask) break; } default: