diff --git a/src/coreclr/jit/codegenwasm.cpp b/src/coreclr/jit/codegenwasm.cpp index 50aaa0ccbf8852..58c0018687da2e 100644 --- a/src/coreclr/jit/codegenwasm.cpp +++ b/src/coreclr/jit/codegenwasm.cpp @@ -2862,14 +2862,8 @@ void CodeGen::genCodeForStoreInd(GenTreeStoreInd* tree) } else // A normal store, not a WriteBarrier store { - var_types type = tree->TypeGet(); - if (type == TYP_SIMD16) - { - // Storing a SIMD16 value emits v128.store, but the data operand is not - // materialized as a v128 (it comes through as an i32), producing an invalid - // module. Bail until SIMD16 store is properly supported. - NYI_WASM_SIMD("SIMD16 store indirect"); - } + var_types type = tree->TypeGet(); + instruction ins = ins_Store(type); // TODO-WASM: Memory barriers diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index c0d4c5a9c61a60..667feb2019022d 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -30774,7 +30774,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const { GenTree* addr = nullptr; -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) +#if defined(TARGET_XARCH) || defined(TARGET_ARM64) || defined(TARGET_WASM) NamedIntrinsic intrinsicId = GetHWIntrinsicId(); HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(intrinsicId); @@ -30964,7 +30964,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const } } #endif // TARGET_XARCH -#endif // TARGET_XARCH || TARGET_ARM64 +#endif // TARGET_XARCH || TARGET_ARM64 || TARGET_WASM if (pAddr != nullptr) { @@ -31031,7 +31031,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryStore(GenTree** pAddr) const { GenTree* addr = nullptr; -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) +#if defined(TARGET_XARCH) || defined(TARGET_ARM64) || defined(TARGET_WASM) NamedIntrinsic intrinsicId = GetHWIntrinsicId(); HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(intrinsicId); @@ -31104,7 +31104,7 @@ bool GenTreeHWIntrinsic::OperIsMemoryStore(GenTree** pAddr) const } } #endif // TARGET_XARCH -#endif // TARGET_XARCH || TARGET_ARM64 +#endif // TARGET_XARCH || TARGET_ARM64 || TARGET_WASM if (pAddr != nullptr) { diff --git a/src/coreclr/jit/hwintrinsic.h b/src/coreclr/jit/hwintrinsic.h index 0b7e966a5e4bf0..dcb7ebe20c8c17 100644 --- a/src/coreclr/jit/hwintrinsic.h +++ b/src/coreclr/jit/hwintrinsic.h @@ -264,6 +264,8 @@ enum HWIntrinsicFlag : uint64_t // The intrinsic supports some sort of containment analysis HW_Flag_SupportsContainment = 0x400, HW_Flag_ReturnsPerElementMask = 0x800, + // The intrinsic has a required immediate operand + HW_Flag_HasImmediateOperand = 0x1000, #else #error Unsupported platform #endif @@ -1003,10 +1005,10 @@ struct HWIntrinsicInfo static bool HasImmediateOperand(NamedIntrinsic id) { -#if defined(TARGET_ARM64) +#if defined(TARGET_ARM64) || defined(TARGET_WASM) const HWIntrinsicFlag flags = lookupFlags(id); return ((flags & HW_Flag_HasImmediateOperand) != 0); -#elif defined(TARGET_XARCH) || defined(TARGET_WASM) +#elif defined(TARGET_XARCH) return lookupCategory(id) == HW_Category_IMM; #else return false; @@ -1329,6 +1331,11 @@ struct HWIntrinsicInfo *imm1Pos = 3; break; } + case NI_PackedSimd_Shuffle: + { + // (v128, v128, shuffle_mask) + break; + } default: { unreached(); @@ -1472,7 +1479,12 @@ struct HWIntrinsic final inline bool needsJumpTableFallback() const { - return !m_node->GetImmOp()->IsCnsIntOrI(); + if (HWIntrinsicInfo::HasImmediateOperand(id)) + { + return !m_node->GetImmOp()->IsCnsIntOrI(); + } + + return false; } uint8_t GetImmediateLaneOperand() const @@ -1488,6 +1500,13 @@ struct HWIntrinsic final return static_cast(lane); } + simd16_t GetImmediateVecOperand() const + { + GenTree* immOp = m_node->GetImmOp(); + assert(immOp->IsCnsVec()); + return immOp->AsVecCon()->gtSimdVal; + } + NamedIntrinsic id; HWIntrinsicCategory category; GenTree* op1; diff --git a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp index f093018d9dcb8f..4939a0bfc8af21 100644 --- a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp @@ -26,9 +26,6 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX // void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { - // emitIns_Lane - // emitIns_Memarg_Lane - const HWIntrinsic info(node); genConsumeMultiOpOperands(node); @@ -40,7 +37,12 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { case HW_Category_SIMD: { - if ((info.id == NI_PackedSimd_Swizzle) && node->Op(2)->isContained()) + if (info.id == NI_PackedSimd_Shuffle) + { + simd16_t shuffleMask = info.GetImmediateVecOperand(); + GetEmitter()->emitIns_V128Imm(ins, shuffleMask.u8); + } + else if ((info.id == NI_PackedSimd_Swizzle) && node->Op(2)->isContained()) { // A constant, fully in-range mask was lowered to an immediate i8x16.shuffle. // prior codegen left the source on the value stack once (the mask @@ -50,7 +52,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) GenTree* src = node->Op(1); regNumber srcReg = GetMultiUseOperandReg(src); GetEmitter()->emitIns_I(INS_local_get, emitActualTypeSize(src), WasmRegToIndex(srcReg)); - GetEmitter()->emitIns_V128Imm(INS_i8x16_shuffle, node->Op(2)->AsVecCon()->gtSimdVal.u8); + GetEmitter()->emitIns_V128Imm(INS_i8x16_shuffle, info.GetImmediateVecOperand().u8); } else { @@ -70,6 +72,32 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) } break; } + case HW_Category_MemoryStore: + case HW_Category_MemoryLoad: + { + emitAttr elemSize = emitActualTypeSize(node->GetSimdBaseType()); + GenTree* addr = nullptr; + bool isMem = node->OperIsMemoryLoad(&addr) || node->OperIsMemoryStore(&addr); + assert(isMem && addr != nullptr); + + regNumber addrReg = GetMultiUseOperandReg(addr); + genEmitNullCheck(addrReg); + + if (info.needsJumpTableFallback()) + { + genHWIntrinsicJumpTableFallback(node, info); + } + else if (HWIntrinsicInfo::HasImmediateOperand(info.id)) + { + GetEmitter()->emitIns_MemargLane(ins, elemSize, 0, info.GetImmediateLaneOperand()); + } + else + { + GetEmitter()->emitIns_I(ins, elemSize, 0); + } + + break; + } default: { NYI_WASM_SIMD("CodeGen::genHWIntrinsic: Unsupported category for table-driven intrinsic"); @@ -142,7 +170,11 @@ void CodeGen::genHWIntrinsicJumpTableFallback(GenTreeHWIntrinsic* node, HWIntrin int simdSize = node->GetSimdSize(); instruction const ins = HWIntrinsicInfo::lookupIns(info.id, info.baseType, m_compiler); int immUpperBound = HWIntrinsicInfo::lookupImmUpperBound(info.id, simdSize, info.baseType); - WasmValueType resultType = ActualTypeToWasmValueType(genActualType(node->TypeGet())); + WasmValueType resultType = WasmValueType::Invalid; + if (!node->TypeIs(TYP_VOID)) + { + resultType = ActualTypeToWasmValueType(genActualType(node->TypeGet())); + } GenTree* immOp = node->GetImmOp(); regNumber immReg = GetMultiUseOperandReg(immOp); @@ -208,6 +240,13 @@ void CodeGen::genHWIntrinsicJumpTableFallback(GenTreeHWIntrinsic* node, HWIntrin GetEmitter()->emitIns_Lane(ins, static_cast(i)); break; } + case HW_Category_MemoryLoad: + case HW_Category_MemoryStore: + { + emitAttr elemSize = emitActualTypeSize(node->GetSimdBaseType()); + GetEmitter()->emitIns_MemargLane(ins, elemSize, 0, static_cast(i)); + break; + } default: { NYI_WASM_SIMD( diff --git a/src/coreclr/jit/hwintrinsiclistwasm.h b/src/coreclr/jit/hwintrinsiclistwasm.h index a4ad1415a32dc2..c639f343656889 100644 --- a/src/coreclr/jit/hwintrinsiclistwasm.h +++ b/src/coreclr/jit/hwintrinsiclistwasm.h @@ -42,13 +42,13 @@ HARDWARE_INTRINSIC(PackedSimd, ConvertToSingle, HARDWARE_INTRINSIC(PackedSimd, ConvertToUInt32Saturate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_i32x4_trunc_sat_u_f32x4, INS_i32x4_trunc_sat_u_f64x2_zero, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Divide, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_div, INS_f64x2_div, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Dot, 16, 2, INS_invalid, INS_invalid, INS_i32x4_dot_i16x8_s, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ExtractScalar, 16, 2, INS_i8x16_extract_lane_s, INS_i8x16_extract_lane_u, INS_i16x8_extract_lane_s, INS_i16x8_extract_lane_u, INS_i32x4_extract_lane, INS_i32x4_extract_lane, INS_i64x2_extract_lane, INS_i64x2_extract_lane, INS_f32x4_extract_lane, INS_f64x2_extract_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ExtractScalar, 16, 2, INS_i8x16_extract_lane_s, INS_i8x16_extract_lane_u, INS_i16x8_extract_lane_s, INS_i16x8_extract_lane_u, INS_i32x4_extract_lane, INS_i32x4_extract_lane, INS_i64x2_extract_lane, INS_i64x2_extract_lane, INS_f32x4_extract_lane, INS_f64x2_extract_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) HARDWARE_INTRINSIC(PackedSimd, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_floor, INS_f64x2_floor, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndInsert, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndSplatVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, LoadScalarVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndInsert, 16, 3, INS_v128_load8_lane, INS_v128_load8_lane, INS_v128_load16_lane, INS_v128_load16_lane, INS_v128_load32_lane, INS_v128_load32_lane, INS_v128_load64_lane, INS_v128_load64_lane, INS_v128_load32_lane, INS_v128_load64_lane, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand) +HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndSplatVector128, 16, 1, INS_v128_load8_splat, INS_v128_load8_splat, INS_v128_load16_splat, INS_v128_load16_splat, INS_v128_load32_splat, INS_v128_load32_splat, INS_v128_load64_splat, INS_v128_load64_splat, INS_v128_load32_splat, INS_v128_load64_splat, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, LoadScalarVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_v128_load32_zero, INS_v128_load32_zero, INS_v128_load64_zero, INS_v128_load64_zero, INS_v128_load32_zero, INS_v128_load64_zero, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, LoadVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, LoadWideningVector128, 16, 1, INS_v128_load8x8_s, INS_v128_load8x8_u, INS_v128_load16x4_s, INS_v128_load16x4_u, INS_v128_load32x2_s, INS_v128_load32x2_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, LoadWideningVector128, 16, 1, INS_v128_load8x8_s, INS_v128_load8x8_u, INS_v128_load16x4_s, INS_v128_load16x4_u, INS_v128_load32x2_s, INS_v128_load32x2_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_MemoryLoad, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Max, 16, 2, INS_i8x16_max_s, INS_i8x16_max_u, INS_i16x8_max_s, INS_i16x8_max_u, INS_i32x4_max_s, INS_i32x4_max_u, INS_invalid, INS_invalid, INS_f32x4_max, INS_f64x2_max, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) HARDWARE_INTRINSIC(PackedSimd, Min, 16, 2, INS_i8x16_min_s, INS_i8x16_min_u, INS_i16x8_min_s, INS_i16x8_min_u, INS_i32x4_min_s, INS_i32x4_min_u, INS_invalid, INS_invalid, INS_f32x4_min, INS_f64x2_min, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) HARDWARE_INTRINSIC(PackedSimd, Multiply, 16, 2, INS_invalid, INS_invalid, INS_i16x8_mul, INS_i16x8_mul, INS_i32x4_mul, INS_i32x4_mul, INS_i64x2_mul, INS_i64x2_mul, INS_f32x4_mul, INS_f64x2_mul, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg|HW_Flag_Commutative) @@ -61,7 +61,7 @@ HARDWARE_INTRINSIC(PackedSimd, Or, HARDWARE_INTRINSIC(PackedSimd, PopCount, 16, 1, INS_invalid, INS_i8x16_popcnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, PseudoMax, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_pmax, INS_f64x2_pmax, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, PseudoMin, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_pmin, INS_f64x2_pmin, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ReplaceScalar, 16, 3, INS_i8x16_replace_lane, INS_i8x16_replace_lane, INS_i16x8_replace_lane, INS_i16x8_replace_lane, INS_i32x4_replace_lane, INS_i32x4_replace_lane, INS_i64x2_replace_lane, INS_i64x2_replace_lane, INS_f32x4_replace_lane, INS_f64x2_replace_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ReplaceScalar, 16, 3, INS_i8x16_replace_lane, INS_i8x16_replace_lane, INS_i16x8_replace_lane, INS_i16x8_replace_lane, INS_i32x4_replace_lane, INS_i32x4_replace_lane, INS_i64x2_replace_lane, INS_i64x2_replace_lane, INS_f32x4_replace_lane, INS_f64x2_replace_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand) HARDWARE_INTRINSIC(PackedSimd, RoundToNearest, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_nearest, INS_f64x2_nearest, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, ShiftLeft, 16, 2, INS_i8x16_shl, INS_i8x16_shl, INS_i16x8_shl, INS_i16x8_shl, INS_i32x4_shl, INS_i32x4_shl, INS_i64x2_shl, INS_i64x2_shl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, ShiftRightArithmetic, 16, 2, INS_i8x16_shr_s, INS_i8x16_shr_s, INS_i16x8_shr_s, INS_i16x8_shr_s, INS_i32x4_shr_s, INS_i32x4_shr_s, INS_i64x2_shr_s, INS_i64x2_shr_s, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) @@ -72,7 +72,7 @@ HARDWARE_INTRINSIC(PackedSimd, SignExtendWideningUpper, HARDWARE_INTRINSIC(PackedSimd, Splat, 16, 1, INS_i8x16_splat, INS_i8x16_splat, INS_i16x8_splat, INS_i16x8_splat, INS_i32x4_splat, INS_i32x4_splat, INS_i64x2_splat, INS_i64x2_splat, INS_f32x4_splat, INS_f64x2_splat, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Sqrt, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_sqrt, INS_f64x2_sqrt, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Store, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(PackedSimd, StoreSelectedScalar, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport|HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(PackedSimd, StoreSelectedScalar, 16, 3, INS_v128_store8_lane, INS_v128_store8_lane, INS_v128_store16_lane, INS_v128_store16_lane, INS_v128_store32_lane, INS_v128_store32_lane, INS_v128_store64_lane, INS_v128_store64_lane, INS_v128_store32_lane, INS_v128_store64_lane, -1, -1, HW_Category_MemoryStore, HW_Flag_BaseTypeFromSecondArg|HW_Flag_HasImmediateOperand) HARDWARE_INTRINSIC(PackedSimd, Subtract, 16, 2, INS_i8x16_sub, INS_i8x16_sub, INS_i16x8_sub, INS_i16x8_sub, INS_i32x4_sub, INS_i32x4_sub, INS_i64x2_sub, INS_i64x2_sub, INS_f32x4_sub, INS_f64x2_sub, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, SubtractSaturate, 16, 2, INS_i8x16_sub_sat_s, INS_i8x16_sub_sat_u, INS_i16x8_sub_sat_s, INS_i16x8_sub_sat_u, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Swizzle, 16, 2, INS_i8x16_swizzle, INS_i8x16_swizzle, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) diff --git a/src/coreclr/jit/hwintrinsicwasm.cpp b/src/coreclr/jit/hwintrinsicwasm.cpp index ea1b948bb8ee69..9475f4aab63cd0 100644 --- a/src/coreclr/jit/hwintrinsicwasm.cpp +++ b/src/coreclr/jit/hwintrinsicwasm.cpp @@ -190,14 +190,6 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } - case NI_PackedSimd_LoadScalarVector128: - case NI_PackedSimd_LoadScalarAndSplatVector128: - case NI_PackedSimd_LoadScalarAndInsert: - case NI_PackedSimd_LoadWideningVector128: - { - break; - } - case NI_PackedSimd_Store: { assert(sig->numArgs == 2); diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index 0313bf4871087d..6f7a3305226387 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -523,6 +523,7 @@ class Lowering final : public Phase #elif defined(TARGET_WASM) GenTree* LowerHWIntrinsicCompareUnsignedLong(GenTreeHWIntrinsic* node); GenTree* LowerHWIntrinsicWithImm(GenTreeHWIntrinsic* node); + GenTree* LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node); void LowerHWIntrinsicSwizzle(GenTreeHWIntrinsic* node); #endif // !TARGET_XARCH && !TARGET_ARM64 GenTree* InsertNewSimdCreateScalarUnsafeNode(var_types type, diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index c55f76236afa90..e0145cf9479b31 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -834,6 +834,13 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) { NamedIntrinsic intrinsic = node->GetHWIntrinsicId(); HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(intrinsic); + bool hasImmOp = HWIntrinsicInfo::HasImmediateOperand(intrinsic); + GenTree* addr = nullptr; + + if (node->OperIsMemoryLoad(&addr) || node->OperIsMemoryStore(&addr)) + { + SetMultiplyUsed(addr DEBUGARG("LowerHWIntrinsic memory address (null check)")); + } switch (intrinsic) { @@ -907,11 +914,27 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) case NI_PackedSimd_ExtractScalar: case NI_PackedSimd_ReplaceScalar: + case NI_PackedSimd_LoadScalarAndInsert: + case NI_PackedSimd_StoreSelectedScalar: { - assert(category == HW_Category_IMM); + assert(hasImmOp); return LowerHWIntrinsicWithImm(node); } + case NI_PackedSimd_Shuffle: + { + return LowerHWIntrinsicNativeShuffle(node); + } + + case NI_PackedSimd_LoadScalarAndSplatVector128: + case NI_PackedSimd_LoadScalarVector128: + case NI_PackedSimd_LoadWideningVector128: + { + // These intrinsics don't require an immediate operand + assert(!hasImmOp); + break; + } + case NI_PackedSimd_Swizzle: { assert(category == HW_Category_SIMD); @@ -1298,6 +1321,96 @@ GenTree* Lowering::LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node) return LowerNode(node); } +// -------------------------------------------------------------------------------- +// LowerHWIntrinsicNativeShuffle: Lowers a PackedSimd Shuffle call with a possibly non-constant mask +// +// Arguments: +// node - The hardware intrinsic node. +// +// Notes: +// If the shuffle mask is a constant vector, it can be contained as an immediate and emitted. Otherwise, +// the shuffle is rewritten into two swizzles for the upper and lower input vectors and combined into the final result. +GenTree* Lowering::LowerHWIntrinsicNativeShuffle(GenTreeHWIntrinsic* node) +{ + assert(node->GetHWIntrinsicId() == NI_PackedSimd_Shuffle); + + GenTree* op1 = node->Op(1); + GenTree* op2 = node->Op(2); + GenTree* shuffleMask = node->Op(3); + var_types resultType = node->TypeGet(); + + // No extra work to do if the shuffle is a constant vector, it can be contained as an immediate and emitted. + if (shuffleMask->IsCnsVec()) + { + ContainCheckHWIntrinsic(node); + return node->gtNext; + } + + // If the shuffle mask is not a constant vector, we will need to rewrite the shuffle into two swizzles: + // 1 to handle elements from the first vector, and 1 to handle elements from the second vector. The two swizzles will then be or'd together to produce the final result. + // We will be constructing IR like the following: + // /--* op1 simd + // +--* originalMask simd + // tmp1 = * HWINTRINSIC simd byte PackedSimd.Swizzle + // upperBnd = * CNS_VEC simd byte <0x10, 0x10, ...> + // /--* originalMask simd + // +--* upperBnd simd + // upperMask = * HWINRINSIC simd byte PackedSimd.Subtract + // /--* op2 simd + // +--* upperMask simd + // tmp2 = * HWINTRINSIC simd byte PackedSimd.Swizzle + // /--* tmp1 simd + // +--* tmp2 simd + // res = * HWINTRINSIC simd byte PackedSimd.Or + + + // Shuffle mask will be used twice, replace with a local + LIR::Use shuffleMaskUse(BlockRange(), &node->Op(3), node); + unsigned int shuffleMaskTmp = shuffleMaskUse.ReplaceWithLclVar(m_compiler); + + // Do a swizzle of the first vector with the original shuffle mask (now loaded from a local), which will produce a vector with the elements from the first vector in the correct order, and zero's for all elements which correspond to the second vector. + GenTree* swizzle1 = m_compiler->gtNewSimdHWIntrinsicNode(resultType, op1, node->Op(3), NI_PackedSimd_Swizzle, TYP_BYTE, 16); + BlockRange().InsertBefore(node, swizzle1); + LowerNode(swizzle1); + + // Subtract 16 from each mask element to mark each element which corresponds to the upper vector as unused, leading to a zero in the result of the swizzle. + GenTreeVecCon* upperBound = m_compiler->gtNewVconNode(shuffleMask->TypeGet()); + upperBound->EvaluateBroadcastInPlace(TYP_BYTE, static_cast(16)); + BlockRange().InsertBefore(node, upperBound); + LowerNode(upperBound); + + // Re-load the original shuffle mask + GenTreeLclVar* shuffleMaskLclVar = m_compiler->gtNewLclVarNode(shuffleMaskTmp, shuffleMask->TypeGet()); + BlockRange().InsertBefore(node, shuffleMaskLclVar); + LowerNode(shuffleMaskLclVar); + + GenTree* upperMask = m_compiler->gtNewSimdHWIntrinsicNode(shuffleMask->TypeGet(), shuffleMaskLclVar, upperBound, NI_PackedSimd_Subtract, TYP_BYTE, 16); + BlockRange().InsertBefore(node, upperMask); + LowerNode(upperMask); + + GenTree* swizzle2 = m_compiler->gtNewSimdHWIntrinsicNode(resultType, op2, upperMask, NI_PackedSimd_Swizzle, TYP_BYTE, 16); + BlockRange().InsertBefore(node, swizzle2); + LowerNode(swizzle2); + + // Since we've left zero's for all the elements which correspond to the upper vector, we can just or the two swizzles together to get the final result. + GenTreeHWIntrinsic* result = m_compiler->gtNewSimdHWIntrinsicNode(resultType, swizzle1, swizzle2, NI_PackedSimd_Or, TYP_INT, 16); + BlockRange().InsertBefore(node, result); + + LIR::Use use; + if (BlockRange().TryGetUse(node, &use)) + { + use.ReplaceWith(result); + } + else + { + result->SetUnusedValue(); + } + + BlockRange().Remove(node); + + return LowerNode(result); +} + //---------------------------------------------------------------------------------------------- // ContainCheckHWIntrinsic: Perform containment analysis for a hardware intrinsic node. // @@ -1306,21 +1419,16 @@ GenTree* Lowering::LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node) // void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) { - HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(node->GetHWIntrinsicId()); - switch (category) + NamedIntrinsic intrinsicId = node->GetHWIntrinsicId(); + if (HWIntrinsicInfo::HasImmediateOperand(intrinsicId) || node->GetHWIntrinsicId() == NI_PackedSimd_Shuffle) { - case HWIntrinsicCategory::HW_Category_IMM: - { - GenTree* immOp = node->GetImmOp(); - if (immOp->IsCnsIntOrI()) - { - MakeSrcContained(node, immOp); - } - break; - } - default: + GenTree* immOp = node->GetImmOp(); + // An immediate operand for SIMD should either be: + // - An integer lane index OR + // - A constant vector (for example, for a shuffle) + if (immOp->IsCnsIntOrI() || immOp->IsCnsVec()) { - break; + MakeSrcContained(node, immOp); } } } diff --git a/src/coreclr/jit/rangecheck.h b/src/coreclr/jit/rangecheck.h index 4880f7696fb070..74c4df51e2ea1c 100644 --- a/src/coreclr/jit/rangecheck.h +++ b/src/coreclr/jit/rangecheck.h @@ -328,10 +328,10 @@ struct RangeOps { if (unsignedAdd) { - bool r1StraddlesZero = r1.IsConstantRange() && (r1.LowerLimit().GetConstant() < 0) && - (r1.UpperLimit().GetConstant() >= 0); - bool r2StraddlesZero = r2.IsConstantRange() && (r2.LowerLimit().GetConstant() < 0) && - (r2.UpperLimit().GetConstant() >= 0); + bool r1StraddlesZero = + r1.IsConstantRange() && (r1.LowerLimit().GetConstant() < 0) && (r1.UpperLimit().GetConstant() >= 0); + bool r2StraddlesZero = + r2.IsConstantRange() && (r2.LowerLimit().GetConstant() < 0) && (r2.UpperLimit().GetConstant() >= 0); if (r1StraddlesZero || r2StraddlesZero) { // Signed intervals that straddle zero are not monotonic when interpreted as unsigned. @@ -355,7 +355,8 @@ struct RangeOps static_assert(CheckedOps::Unsigned == true); // For unsigned adds, require both unsigned and signed endpoint sums to not overflow. bool requestedAddOverflows = CheckedOps::AddOverflows(a.GetConstant(), b.GetConstant(), unsignedAdd); - bool signedEndpointOverflows = unsignedAdd && CheckedOps::AddOverflows(a.GetConstant(), b.GetConstant(), CheckedOps::Signed); + bool signedEndpointOverflows = + unsignedAdd && CheckedOps::AddOverflows(a.GetConstant(), b.GetConstant(), CheckedOps::Signed); if (!requestedAddOverflows && !signedEndpointOverflows) { if (a.IsConstant() && b.IsConstant()) diff --git a/src/coreclr/jit/stacklevelsetter.cpp b/src/coreclr/jit/stacklevelsetter.cpp index c33de2c033d374..ccba1b54334f46 100644 --- a/src/coreclr/jit/stacklevelsetter.cpp +++ b/src/coreclr/jit/stacklevelsetter.cpp @@ -273,7 +273,18 @@ void StackLevelSetter::SetThrowHelperBlocks(GenTree* node, BasicBlock* block) } } break; -#endif // defined(FEATURE_HW_INTRINSICS) && defined(TARGET_XARCH) +#elif defined(FEATURE_HW_INTRINSICS) && defined(TARGET_WASM) + case GT_HWINTRINSIC: + { + HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(node->AsHWIntrinsic()->GetHWIntrinsicId()); + if (category == HW_Category_MemoryLoad || category == HW_Category_MemoryStore) + { + SetThrowHelperBlock(SCK_NULL_CHECK, block); + } + } + break; + +#endif // defined(FEATURE_HW_INTRINSICS) && (defined(TARGET_XARCH) || defined(TARGET_WASM)) case GT_INDEX_ADDR: if (node->AsIndexAddr()->IsBoundsChecked()) diff --git a/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs b/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs index b850be9c523cc6..9d978d0d4230d8 100644 --- a/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs +++ b/src/tests/JIT/HardwareIntrinsics/Wasm/PackedSimd/PackedSimdTests.cs @@ -1074,6 +1074,48 @@ public static unsafe void LoadScalarAndSplatTest() Assert.Equal(Vector128.Create(3.14f, 3.14f, 3.14f, 3.14f), floatSplat); } + [Fact] + public static unsafe void LoadStoreNullCheckTest() + { + Assert.Throws(() => LoadScalarAndSplatVector128(null)); + Assert.Throws(() => LoadScalarVector128(null)); + Assert.Throws(() => LoadWideningVector128(null)); + Assert.Throws(() => LoadScalarAndInsert(null, 2)); + Assert.Throws(() => StoreSelectedScalar(null, 2)); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe Vector128 LoadScalarAndSplatVector128(int* address) + { + return PackedSimd.LoadScalarAndSplatVector128(address); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe Vector128 LoadScalarVector128(int* address) + { + return PackedSimd.LoadScalarVector128(address); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe Vector128 LoadWideningVector128(sbyte* address) + { + return PackedSimd.LoadWideningVector128(address); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe Vector128 LoadScalarAndInsert(int* address, byte index) + { + Vector128 vector = Vector128.Create(1, 2, 3, 4); + return PackedSimd.LoadScalarAndInsert(address, vector, index); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static unsafe void StoreSelectedScalar(int* address, byte index) + { + Vector128 vector = Vector128.Create(1, 2, 3, 4); + PackedSimd.StoreSelectedScalar(address, vector, index); + } + [Fact] public static unsafe void LoadWideningTest() {